CI: Maintenance

- Rename .travis dir to .ci, since it isn't just for Travis
- Convert Linux build scripts to posix sh
- Clean up some scripts per shellcheck

.ci/build-linux.sh

@@ -1,23 +1,25 @@
-#!/bin/env bash -ex
+#!/bin/sh -ex
-shopt -s nocasematch
 
 # Setup Qt variables
 export QT_BASE_DIR=/opt/qt${QTVERMIN}
 export PATH=$QT_BASE_DIR/bin:$PATH
 export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib
 
-cd rpcs3
+cd rpcs3 || exit 1
 
-git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/libusb 3rdparty/FAudio Vulkan/glslang 3rdparty/curl 3rdparty/wolfssl
+# Pull all the submodules except llvm, since it is built separately and we just download that build
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
 
 # Download pre-compiled llvm libs
 curl -sLO https://github.com/RPCS3/llvm-mirror/releases/download/custom-build/llvmlibs-linux.tar.gz
 mkdir llvmlibs
 tar -xzf ./llvmlibs-linux.tar.gz -C llvmlibs
 
-mkdir build ; cd build
+mkdir build && cd build || exit 1
 
-if [ $COMPILER = "gcc" ]; then
+if [ "$COMPILER" = "gcc" ]; then
 	# These are set in the dockerfile
 	export CC=${GCC_BINARY}
 	export CXX=${GXX_BINARY}
@@ -44,8 +46,13 @@ ninja; build_status=$?;
 
 cd ..
 
-# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines)
+# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines).
-# BUILD_REASON is an Azure Pipeline variable, and we want to deploy when using Azure Pipelines
+# Travis only deploys on master, and it publishes to GitHub releases. Azure publishes PRs as artifacts
-if [[ $build_status -eq 0 && ( -n "$BUILD_REASON" || ( "$TRAVIS_BRANCH" = "master" && "$TRAVIS_PULL_REQUEST" = false ) ) ]]; then
+# only.
-	/bin/bash -ex .travis/deploy-linux.bash
+{   [ "$IS_AZURE" = "true" ] ||
+	{ [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; };
+} && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false"
+
+if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then
+	.ci/deploy-linux.sh
 fi

.ci/build-mac.sh

@@ -1,3 +1,5 @@
+#!/bin/sh -ex
+
 export CCACHE_SLOPPINESS=pch_defines,time_macros
 export CMAKE_PREFIX_PATH=/usr/local/opt/qt5/
 export PATH="/usr/local/opt/ccache/libexec:$PATH"
@@ -8,15 +10,18 @@ unzip -: gfx-portability-macos-latest.zip
 curl -sLO https://github.com/KhronosGroup/Vulkan-Headers/archive/sdk-1.1.106.0.zip
 unzip -: sdk-*.zip
 mkdir vulkan-sdk
-ln -s ${PWD}/Vulkan-Headers*/include vulkan-sdk/include
+ln -s "${PWD}"/Vulkan-Headers*/include vulkan-sdk/include
 mkdir vulkan-sdk/lib
 cp target/release/libportability.dylib vulkan-sdk/lib/libVulkan.dylib
 # Let macdeployqt locate and install Vulkan library
-install_name_tool -id ${PWD}/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
+install_name_tool -id "${PWD}"/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
-export VULKAN_SDK=${PWD}/vulkan-sdk
+export VULKAN_SDK="${PWD}/vulkan-sdk"
 
-git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/libusb 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/FAudio Vulkan/glslang
+# Pull all the submodules except llvm
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
 
-mkdir build; cd build
+mkdir build && cd build || exit 1
 cmake .. -DWITH_LLVM=OFF -DUSE_NATIVE_INSTRUCTIONS=OFF -G Ninja
 ninja

.ci/deploy-linux.sh

@@ -1,16 +1,19 @@
-#!/bin/env bash -ex
+#!/bin/sh -ex
-shopt -s nocasematch
+
-cd build
+cd build || exit 1
+
 if [ "$DEPLOY_APPIMAGE" = "true" ]; then
 	DESTDIR=appdir ninja install
-	curl -sLO "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage"
+	QT_APPIMAGE="linuxdeployqt.AppImage"
-	chmod a+x linuxdeployqt*.AppImage
+
-	./linuxdeployqt*.AppImage --appimage-extract
+    curl -sL "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" > "$QT_APPIMAGE"
+    chmod a+x "$QT_APPIMAGE"
+    "./$QT_APPIMAGE" --appimage-extract
     ./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs
     ls ./appdir/usr/lib/
     rm -r ./appdir/usr/share/doc
     rm ./appdir/usr/lib/libxcb*
-	cp $(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1) ./appdir/usr/lib/libnsl.so.1
+    cp "$(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1)" ./appdir/usr/lib/libnsl.so.1
     export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH}
 
 	# Embed newer libstdc++ for distros that don't come with it (ubuntu 16.04)
@@ -32,25 +35,22 @@ if [ "$DEPLOY_APPIMAGE" = "true" ]; then
 	# Package it up and send it off
 	./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir
 	ls
-	COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[{,] '{printf "%d.%d.%d", $2, $3, $4}')"
+
+	COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')"
 	COMM_COUNT="$(git rev-list --count HEAD)"
+	COMM_HASH="$(git rev-parse --short=8 HEAD)"
+	RPCS3_APPIMAGE="rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMM_HASH}_linux64.AppImage"
+
 	curl -sLO https://github.com/hcorion/uploadtool/raw/master/upload.sh
-	
+	mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE"
-	if [[ -n "$BUILD_SOURCEVERSION" ]]; then
-		COMMIT_HASH=$BUILD_SOURCEVERSION
-	elif [[ -n "$TRAVIS_COMMIT" ]]; then
-		COMMIT_HASH=$TRAVIS_COMMIT
-	fi
-	
-	mv ./RPCS3*.AppImage rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMMIT_HASH:0:8}_linux64.AppImage
 
 	# If we're building using Azure Pipelines, let's copy over the AppImage artifact
-	if [[ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]]; then
+	if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then
-		cp ./rpcs3*.AppImage ~/artifacts
+		cp "$RPCS3_APPIMAGE" ~/artifacts
 	fi
 
-	FILESIZE=($(stat -c %s ./rpcs3*.AppImage))
+	FILESIZE=$(stat -c %s ./rpcs3*.AppImage)
-	SHA256SUM=($(sha256sum ./rpcs3*.AppImage))
+	SHA256SUM=$(sha256sum ./rpcs3*.AppImage)
 	if [ -n "$GITHUB_TOKEN" ]; then
 		unset TRAVIS_REPO_SLUG
 		REPO_SLUG=RPCS3/rpcs3-binaries-linux \

.ci/export-azure-vars.sh

@@ -10,4 +10,4 @@ while IFS='=' read -r key val; do
     # Skip over lines containing comments.
     [ "${key##\#*}" ] || continue
     echo "##vso[task.setvariable variable=$key]$val"
-done < ".travis/azure-vars.env"
+done < ".ci/azure-vars.env"

.ci/setup-windows.sh

@@ -111,6 +111,6 @@ fi
 # BUILD is the name of the release artifact
 # AVVER is used for GitHub releases, it is the version number.
 BRANCH="${REPO_NAME}/${REPO_BRANCH}"
-echo "BRANCH=$BRANCH" > .travis/azure-vars.env
+echo "BRANCH=$BRANCH" > .ci/azure-vars.env
-echo "BUILD=$BUILD" >> .travis/azure-vars.env
+echo "BUILD=$BUILD" >> .ci/azure-vars.env
-echo "AVVER=$AVVER" >> .travis/azure-vars.env
+echo "AVVER=$AVVER" >> .ci/azure-vars.env

.ci/travis.env

@@ -3,6 +3,7 @@ TRAVIS_PULL_REQUEST
 TRAVIS_BRANCH
 TRAVIS_COMMIT
 # Variables set by Azure Pipelines
+IS_AZURE
 BUILD_REASON
 BUILD_SOURCEVERSION
 BUILD_ARTIFACTSTAGINGDIRECTORY

.travis.yml

@@ -11,7 +11,7 @@ jobs:
           cache: ccache
           compiler: gcc
           install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
         - os: linux
           dist: xenial
           env:
@@ -21,7 +21,7 @@ jobs:
           cache: ccache
           compiler: clang
           install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
         # - os: osx
         #   osx_image: xcode11.3
         #   addons:
@@ -31,7 +31,7 @@ jobs:
         #             - glew
         #             - ninja
         #             - qt
-        #   script: "/bin/bash -ex .travis/build-mac.bash"
+        #   script: "/bin/bash -ex .ci/build-mac.sh"
         #   cache: ccache
     allow_failures:
         - os: osx

azure-pipelines.yml

@@ -20,6 +20,7 @@ jobs:
         DEPLOY_APPIMAGE: true
   variables:
     CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    IS_AZURE: true
   pool:
     vmImage: 'ubuntu-latest'
   steps:
@@ -33,11 +34,11 @@ jobs:
       docker pull --quiet rpcs3/rpcs3-travis-xenial:1.2
       docker run                      \
         -v $(pwd):/rpcs3              \
-        --env-file .travis/travis.env \
+        --env-file .ci/travis.env \
         -v $CCACHE_DIR:/root/.ccache  \
         -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/root/artifacts \
         rpcs3/rpcs3-travis-xenial:1.2 \
-        /bin/bash -ex /rpcs3/.travis/build-linux.bash
+        /rpcs3/.ci/build-linux.sh
     displayName: Docker setup and build
 
   - publish: $(Build.ArtifactStagingDirectory)
@@ -64,10 +65,10 @@ jobs:
         path: $(CACHE_DIR)
       displayName: Cache
 
-    - bash: .travis/setup-windows.sh
+    - bash: .ci/setup-windows.sh
       displayName: Download and unpack dependencies
 
-    - bash: .travis/export-azure-vars.sh
+    - bash: .ci/export-azure-vars.sh
       displayName: Export Variables
 
     - task: VSBuild@1
@@ -78,7 +79,7 @@ jobs:
         configuration: 'Release - LLVM'
       displayName: Compile RPCS3
 
-    - bash: .travis/deploy-windows.sh
+    - bash: .ci/deploy-windows.sh
       displayName: Pack up build artifacts
 
     - publish: $(Build.ArtifactStagingDirectory)