CI: Maintenance

- Rename .travis dir to .ci, since it isn't just for Travis - Convert Linux build scripts to posix sh - Clean up some scripts per shellcheck
2024-11-21 18:22:33 +01:00 · 2020-04-06 19:53:23 -07:00 · 2020-04-06 19:53:23 -07:00 · c4a21438ad
commit c4a21438ad
parent c2ae8be3eb
9 changed files with 71 additions and 57 deletions
--- a/.travis/build-linux.bash
+++ b/.travis/build-linux.bash
@ -1,23 +1,25 @@
-#!/bin/env bash -ex
-shopt -s nocasematch
+#!/bin/sh -ex

 # Setup Qt variables
 export QT_BASE_DIR=/opt/qt${QTVERMIN}
 export PATH=$QT_BASE_DIR/bin:$PATH
 export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib

-cd rpcs3
+cd rpcs3 || exit 1

-git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/libusb 3rdparty/FAudio Vulkan/glslang 3rdparty/curl 3rdparty/wolfssl
+# Pull all the submodules except llvm, since it is built separately and we just download that build
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)

 # Download pre-compiled llvm libs
 curl -sLO https://github.com/RPCS3/llvm-mirror/releases/download/custom-build/llvmlibs-linux.tar.gz
 mkdir llvmlibs
 tar -xzf ./llvmlibs-linux.tar.gz -C llvmlibs

-mkdir build ; cd build
+mkdir build && cd build || exit 1

-if [ $COMPILER = "gcc" ]; then
+if [ "$COMPILER" = "gcc" ]; then
 	# These are set in the dockerfile
 	export CC=${GCC_BINARY}
 	export CXX=${GXX_BINARY}
@ -44,8 +46,13 @@ ninja; build_status=$?;

 cd ..

-# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines)
-# BUILD_REASON is an Azure Pipeline variable, and we want to deploy when using Azure Pipelines
-if [[ $build_status -eq 0 && ( -n "$BUILD_REASON" || ( "$TRAVIS_BRANCH" = "master" && "$TRAVIS_PULL_REQUEST" = false ) ) ]]; then
-	/bin/bash -ex .travis/deploy-linux.bash
+# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines).
+# Travis only deploys on master, and it publishes to GitHub releases. Azure publishes PRs as artifacts
+# only.
+{   [ "$IS_AZURE" = "true" ] ||
+	{ [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; };
+} && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false"
+
+if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then
+	.ci/deploy-linux.sh
 fi
--- a/.travis/build-mac.bash
+++ b/.travis/build-mac.bash
@ -1,3 +1,5 @@
+#!/bin/sh -ex
+
 export CCACHE_SLOPPINESS=pch_defines,time_macros
 export CMAKE_PREFIX_PATH=/usr/local/opt/qt5/
 export PATH="/usr/local/opt/ccache/libexec:$PATH"
@ -8,15 +10,18 @@ unzip -: gfx-portability-macos-latest.zip
 curl -sLO https://github.com/KhronosGroup/Vulkan-Headers/archive/sdk-1.1.106.0.zip
 unzip -: sdk-*.zip
 mkdir vulkan-sdk
-ln -s ${PWD}/Vulkan-Headers*/include vulkan-sdk/include
+ln -s "${PWD}"/Vulkan-Headers*/include vulkan-sdk/include
 mkdir vulkan-sdk/lib
 cp target/release/libportability.dylib vulkan-sdk/lib/libVulkan.dylib
 # Let macdeployqt locate and install Vulkan library
-install_name_tool -id ${PWD}/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
-export VULKAN_SDK=${PWD}/vulkan-sdk
+install_name_tool -id "${PWD}"/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
+export VULKAN_SDK="${PWD}/vulkan-sdk"

-git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/libusb 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/FAudio Vulkan/glslang
+# Pull all the submodules except llvm
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)

-mkdir build; cd build
+mkdir build && cd build || exit 1
 cmake .. -DWITH_LLVM=OFF -DUSE_NATIVE_INSTRUCTIONS=OFF -G Ninja
 ninja
--- a/.travis/deploy-linux.bash
+++ b/.travis/deploy-linux.bash
@ -1,17 +1,20 @@
-#!/bin/env bash -ex
-shopt -s nocasematch
-cd build
+#!/bin/sh -ex
+
+cd build || exit 1
+
 if [ "$DEPLOY_APPIMAGE" = "true" ]; then
 	DESTDIR=appdir ninja install
-	curl -sLO "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage"
-	chmod a+x linuxdeployqt*.AppImage
-	./linuxdeployqt*.AppImage --appimage-extract
-	./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs
-	ls ./appdir/usr/lib/
-	rm -r ./appdir/usr/share/doc
-	rm ./appdir/usr/lib/libxcb*
-	cp $(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1) ./appdir/usr/lib/libnsl.so.1
-	export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH}
+	QT_APPIMAGE="linuxdeployqt.AppImage"
+
+    curl -sL "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" > "$QT_APPIMAGE"
+    chmod a+x "$QT_APPIMAGE"
+    "./$QT_APPIMAGE" --appimage-extract
+    ./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs
+    ls ./appdir/usr/lib/
+    rm -r ./appdir/usr/share/doc
+    rm ./appdir/usr/lib/libxcb*
+    cp "$(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1)" ./appdir/usr/lib/libnsl.so.1
+    export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH}

 	# Embed newer libstdc++ for distros that don't come with it (ubuntu 16.04)
 	mkdir -p appdir/usr/optional/ ; mkdir -p appdir/usr/optional/libstdc++/
@ -20,37 +23,34 @@ if [ "$DEPLOY_APPIMAGE" = "true" ]; then
 	curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/AppRun-patched-x86_64 -o ./appdir/AppRun
 	chmod a+x ./appdir/AppRun
 	curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/exec-x86_64.so -o ./appdir/usr/optional/exec.so
-	
+
 	# Compile checker binary for AppImageKit-checkrt
-	
+
 	# This may need updating if you update the compiler or rpcs3 uses newer c++ features
 	# See https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/config/abi/pre/gnu.ver
 	# for which definitions correlate to which CXXABI version.
 	# Currently we target a minimum of GLIBCXX_3.4.26 and CXXABI_1.3.11
 	printf "#include <bits/stdc++.h>\nint main(){std::make_exception_ptr(0);std::pmr::get_default_resource();}" | $CXX -x c++ -std=c++2a -o ./appdir/usr/optional/checker -
-	
+
 	# Package it up and send it off
 	./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir
 	ls
-	COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[{,] '{printf "%d.%d.%d", $2, $3, $4}')"
+
+	COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')"
 	COMM_COUNT="$(git rev-list --count HEAD)"
+	COMM_HASH="$(git rev-parse --short=8 HEAD)"
+	RPCS3_APPIMAGE="rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMM_HASH}_linux64.AppImage"
+
 	curl -sLO https://github.com/hcorion/uploadtool/raw/master/upload.sh
-	
-	if [[ -n "$BUILD_SOURCEVERSION" ]]; then
-		COMMIT_HASH=$BUILD_SOURCEVERSION
-	elif [[ -n "$TRAVIS_COMMIT" ]]; then
-		COMMIT_HASH=$TRAVIS_COMMIT
-	fi
-	
-	mv ./RPCS3*.AppImage rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMMIT_HASH:0:8}_linux64.AppImage
-	
+	mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE"
+
 	# If we're building using Azure Pipelines, let's copy over the AppImage artifact
-	if [[ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]]; then
-		cp ./rpcs3*.AppImage ~/artifacts
+	if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then
+		cp "$RPCS3_APPIMAGE" ~/artifacts
 	fi
-	
-	FILESIZE=($(stat -c %s ./rpcs3*.AppImage))
-	SHA256SUM=($(sha256sum ./rpcs3*.AppImage))
+
+	FILESIZE=$(stat -c %s ./rpcs3*.AppImage)
+	SHA256SUM=$(sha256sum ./rpcs3*.AppImage)
 	if [ -n "$GITHUB_TOKEN" ]; then
 		unset TRAVIS_REPO_SLUG
 		REPO_SLUG=RPCS3/rpcs3-binaries-linux \
--- a/.travis/deploy-windows.sh
+++ b/.travis/deploy-windows.sh
--- a/.travis/export-azure-vars.sh
+++ b/.travis/export-azure-vars.sh
@ -10,4 +10,4 @@ while IFS='=' read -r key val; do
    # Skip over lines containing comments.
    [ "${key##\#*}" ] || continue
    echo "##vso[task.setvariable variable=$key]$val"
-done < ".travis/azure-vars.env"
+done < ".ci/azure-vars.env"
--- a/.travis/setup-windows.sh
+++ b/.travis/setup-windows.sh
@ -111,6 +111,6 @@ fi
 # BUILD is the name of the release artifact
 # AVVER is used for GitHub releases, it is the version number.
 BRANCH="${REPO_NAME}/${REPO_BRANCH}"
-echo "BRANCH=$BRANCH" > .travis/azure-vars.env
-echo "BUILD=$BUILD" >> .travis/azure-vars.env
-echo "AVVER=$AVVER" >> .travis/azure-vars.env
+echo "BRANCH=$BRANCH" > .ci/azure-vars.env
+echo "BUILD=$BUILD" >> .ci/azure-vars.env
+echo "AVVER=$AVVER" >> .ci/azure-vars.env
--- a/.travis/travis.env
+++ b/.travis/travis.env
@ -3,6 +3,7 @@ TRAVIS_PULL_REQUEST
 TRAVIS_BRANCH
 TRAVIS_COMMIT
 # Variables set by Azure Pipelines
+IS_AZURE
 BUILD_REASON
 BUILD_SOURCEVERSION
 BUILD_ARTIFACTSTAGINGDIRECTORY
--- a/.travis.yml
+++ b/.travis.yml
@ -11,7 +11,7 @@ jobs:
          cache: ccache
          compiler: gcc
          install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
        - os: linux
          dist: xenial
          env:
@ -21,7 +21,7 @@ jobs:
          cache: ccache
          compiler: clang
          install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
        # - os: osx
        #   osx_image: xcode11.3
        #   addons:
@ -31,7 +31,7 @@ jobs:
        #             - glew
        #             - ninja
        #             - qt
-        #   script: "/bin/bash -ex .travis/build-mac.bash"
+        #   script: "/bin/bash -ex .ci/build-mac.sh"
        #   cache: ccache
    allow_failures:
        - os: osx
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -20,6 +20,7 @@ jobs:
        DEPLOY_APPIMAGE: true
  variables:
    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    IS_AZURE: true
  pool:
    vmImage: 'ubuntu-latest'
  steps:
@ -33,11 +34,11 @@ jobs:
      docker pull --quiet rpcs3/rpcs3-travis-xenial:1.2
      docker run                      \
        -v $(pwd):/rpcs3              \
-        --env-file .travis/travis.env \
+        --env-file .ci/travis.env \
        -v $CCACHE_DIR:/root/.ccache  \
        -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/root/artifacts \
        rpcs3/rpcs3-travis-xenial:1.2 \
-        /bin/bash -ex /rpcs3/.travis/build-linux.bash
+        /rpcs3/.ci/build-linux.sh
    displayName: Docker setup and build

  - publish: $(Build.ArtifactStagingDirectory)
@ -64,10 +65,10 @@ jobs:
        path: $(CACHE_DIR)
      displayName: Cache

-    - bash: .travis/setup-windows.sh
+    - bash: .ci/setup-windows.sh
      displayName: Download and unpack dependencies

-    - bash: .travis/export-azure-vars.sh
+    - bash: .ci/export-azure-vars.sh
      displayName: Export Variables

    - task: VSBuild@1
@ -78,7 +79,7 @@ jobs:
        configuration: 'Release - LLVM'
      displayName: Compile RPCS3

-    - bash: .travis/deploy-windows.sh
+    - bash: .ci/deploy-windows.sh
      displayName: Pack up build artifacts

    - publish: $(Build.ArtifactStagingDirectory)