From c4a21438ad1fe5f89714ba9af33b841714ef4cbd Mon Sep 17 00:00:00 2001 From: JohnHolmesII Date: Mon, 6 Apr 2020 19:53:23 -0700 Subject: [PATCH] CI: Maintenance - Rename .travis dir to .ci, since it isn't just for Travis - Convert Linux build scripts to posix sh - Clean up some scripts per shellcheck --- .../build-linux.bash => .ci/build-linux.sh | 27 +++++---- .travis/build-mac.bash => .ci/build-mac.sh | 15 +++-- .../deploy-linux.bash => .ci/deploy-linux.sh | 60 +++++++++---------- {.travis => .ci}/deploy-windows.sh | 0 {.travis => .ci}/export-azure-vars.sh | 2 +- {.travis => .ci}/setup-windows.sh | 6 +- {.travis => .ci}/travis.env | 1 + .travis.yml | 6 +- azure-pipelines.yml | 11 ++-- 9 files changed, 71 insertions(+), 57 deletions(-) rename .travis/build-linux.bash => .ci/build-linux.sh (60%) mode change 100644 => 100755 rename .travis/build-mac.bash => .ci/build-mac.sh (59%) mode change 100644 => 100755 rename .travis/deploy-linux.bash => .ci/deploy-linux.sh (64%) mode change 100644 => 100755 rename {.travis => .ci}/deploy-windows.sh (100%) mode change 100644 => 100755 rename {.travis => .ci}/export-azure-vars.sh (92%) mode change 100644 => 100755 rename {.travis => .ci}/setup-windows.sh (96%) mode change 100644 => 100755 rename {.travis => .ci}/travis.env (97%) diff --git a/.travis/build-linux.bash b/.ci/build-linux.sh old mode 100644 new mode 100755 similarity index 60% rename from .travis/build-linux.bash rename to .ci/build-linux.sh index 0aadc749f4..688fb12491 --- a/.travis/build-linux.bash +++ b/.ci/build-linux.sh @@ -1,23 +1,25 @@ -#!/bin/env bash -ex -shopt -s nocasematch +#!/bin/sh -ex # Setup Qt variables export QT_BASE_DIR=/opt/qt${QTVERMIN} export PATH=$QT_BASE_DIR/bin:$PATH export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib -cd rpcs3 +cd rpcs3 || exit 1 -git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/libusb 3rdparty/FAudio Vulkan/glslang 3rdparty/curl 3rdparty/wolfssl +# Pull all the submodules except llvm, since it is built separately and we just download that build +# Note: Tried to use git submodule status, but it takes over 20 seconds +# shellcheck disable=SC2046 +git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules) # Download pre-compiled llvm libs curl -sLO https://github.com/RPCS3/llvm-mirror/releases/download/custom-build/llvmlibs-linux.tar.gz mkdir llvmlibs tar -xzf ./llvmlibs-linux.tar.gz -C llvmlibs -mkdir build ; cd build +mkdir build && cd build || exit 1 -if [ $COMPILER = "gcc" ]; then +if [ "$COMPILER" = "gcc" ]; then # These are set in the dockerfile export CC=${GCC_BINARY} export CXX=${GXX_BINARY} @@ -44,8 +46,13 @@ ninja; build_status=$?; cd .. -# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines) -# BUILD_REASON is an Azure Pipeline variable, and we want to deploy when using Azure Pipelines -if [[ $build_status -eq 0 && ( -n "$BUILD_REASON" || ( "$TRAVIS_BRANCH" = "master" && "$TRAVIS_PULL_REQUEST" = false ) ) ]]; then - /bin/bash -ex .travis/deploy-linux.bash +# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines). +# Travis only deploys on master, and it publishes to GitHub releases. Azure publishes PRs as artifacts +# only. +{ [ "$IS_AZURE" = "true" ] || + { [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; }; +} && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false" + +if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then + .ci/deploy-linux.sh fi diff --git a/.travis/build-mac.bash b/.ci/build-mac.sh old mode 100644 new mode 100755 similarity index 59% rename from .travis/build-mac.bash rename to .ci/build-mac.sh index bc3b7d430b..5c2cf57e19 --- a/.travis/build-mac.bash +++ b/.ci/build-mac.sh @@ -1,3 +1,5 @@ +#!/bin/sh -ex + export CCACHE_SLOPPINESS=pch_defines,time_macros export CMAKE_PREFIX_PATH=/usr/local/opt/qt5/ export PATH="/usr/local/opt/ccache/libexec:$PATH" @@ -8,15 +10,18 @@ unzip -: gfx-portability-macos-latest.zip curl -sLO https://github.com/KhronosGroup/Vulkan-Headers/archive/sdk-1.1.106.0.zip unzip -: sdk-*.zip mkdir vulkan-sdk -ln -s ${PWD}/Vulkan-Headers*/include vulkan-sdk/include +ln -s "${PWD}"/Vulkan-Headers*/include vulkan-sdk/include mkdir vulkan-sdk/lib cp target/release/libportability.dylib vulkan-sdk/lib/libVulkan.dylib # Let macdeployqt locate and install Vulkan library -install_name_tool -id ${PWD}/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib -export VULKAN_SDK=${PWD}/vulkan-sdk +install_name_tool -id "${PWD}"/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib +export VULKAN_SDK="${PWD}/vulkan-sdk" -git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/libusb 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/FAudio Vulkan/glslang +# Pull all the submodules except llvm +# Note: Tried to use git submodule status, but it takes over 20 seconds +# shellcheck disable=SC2046 +git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules) -mkdir build; cd build +mkdir build && cd build || exit 1 cmake .. -DWITH_LLVM=OFF -DUSE_NATIVE_INSTRUCTIONS=OFF -G Ninja ninja diff --git a/.travis/deploy-linux.bash b/.ci/deploy-linux.sh old mode 100644 new mode 100755 similarity index 64% rename from .travis/deploy-linux.bash rename to .ci/deploy-linux.sh index 13575ac15d..bffd7ae446 --- a/.travis/deploy-linux.bash +++ b/.ci/deploy-linux.sh @@ -1,17 +1,20 @@ -#!/bin/env bash -ex -shopt -s nocasematch -cd build +#!/bin/sh -ex + +cd build || exit 1 + if [ "$DEPLOY_APPIMAGE" = "true" ]; then DESTDIR=appdir ninja install - curl -sLO "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" - chmod a+x linuxdeployqt*.AppImage - ./linuxdeployqt*.AppImage --appimage-extract - ./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs - ls ./appdir/usr/lib/ - rm -r ./appdir/usr/share/doc - rm ./appdir/usr/lib/libxcb* - cp $(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1) ./appdir/usr/lib/libnsl.so.1 - export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH} + QT_APPIMAGE="linuxdeployqt.AppImage" + + curl -sL "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" > "$QT_APPIMAGE" + chmod a+x "$QT_APPIMAGE" + "./$QT_APPIMAGE" --appimage-extract + ./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs + ls ./appdir/usr/lib/ + rm -r ./appdir/usr/share/doc + rm ./appdir/usr/lib/libxcb* + cp "$(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1)" ./appdir/usr/lib/libnsl.so.1 + export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH} # Embed newer libstdc++ for distros that don't come with it (ubuntu 16.04) mkdir -p appdir/usr/optional/ ; mkdir -p appdir/usr/optional/libstdc++/ @@ -20,37 +23,34 @@ if [ "$DEPLOY_APPIMAGE" = "true" ]; then curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/AppRun-patched-x86_64 -o ./appdir/AppRun chmod a+x ./appdir/AppRun curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/exec-x86_64.so -o ./appdir/usr/optional/exec.so - + # Compile checker binary for AppImageKit-checkrt - + # This may need updating if you update the compiler or rpcs3 uses newer c++ features # See https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/config/abi/pre/gnu.ver # for which definitions correlate to which CXXABI version. # Currently we target a minimum of GLIBCXX_3.4.26 and CXXABI_1.3.11 printf "#include \nint main(){std::make_exception_ptr(0);std::pmr::get_default_resource();}" | $CXX -x c++ -std=c++2a -o ./appdir/usr/optional/checker - - + # Package it up and send it off ./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir ls - COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[{,] '{printf "%d.%d.%d", $2, $3, $4}')" + + COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')" COMM_COUNT="$(git rev-list --count HEAD)" + COMM_HASH="$(git rev-parse --short=8 HEAD)" + RPCS3_APPIMAGE="rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMM_HASH}_linux64.AppImage" + curl -sLO https://github.com/hcorion/uploadtool/raw/master/upload.sh - - if [[ -n "$BUILD_SOURCEVERSION" ]]; then - COMMIT_HASH=$BUILD_SOURCEVERSION - elif [[ -n "$TRAVIS_COMMIT" ]]; then - COMMIT_HASH=$TRAVIS_COMMIT - fi - - mv ./RPCS3*.AppImage rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMMIT_HASH:0:8}_linux64.AppImage - + mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE" + # If we're building using Azure Pipelines, let's copy over the AppImage artifact - if [[ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]]; then - cp ./rpcs3*.AppImage ~/artifacts + if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then + cp "$RPCS3_APPIMAGE" ~/artifacts fi - - FILESIZE=($(stat -c %s ./rpcs3*.AppImage)) - SHA256SUM=($(sha256sum ./rpcs3*.AppImage)) + + FILESIZE=$(stat -c %s ./rpcs3*.AppImage) + SHA256SUM=$(sha256sum ./rpcs3*.AppImage) if [ -n "$GITHUB_TOKEN" ]; then unset TRAVIS_REPO_SLUG REPO_SLUG=RPCS3/rpcs3-binaries-linux \ diff --git a/.travis/deploy-windows.sh b/.ci/deploy-windows.sh old mode 100644 new mode 100755 similarity index 100% rename from .travis/deploy-windows.sh rename to .ci/deploy-windows.sh diff --git a/.travis/export-azure-vars.sh b/.ci/export-azure-vars.sh old mode 100644 new mode 100755 similarity index 92% rename from .travis/export-azure-vars.sh rename to .ci/export-azure-vars.sh index a54c2b7bff..7ab0515200 --- a/.travis/export-azure-vars.sh +++ b/.ci/export-azure-vars.sh @@ -10,4 +10,4 @@ while IFS='=' read -r key val; do # Skip over lines containing comments. [ "${key##\#*}" ] || continue echo "##vso[task.setvariable variable=$key]$val" -done < ".travis/azure-vars.env" +done < ".ci/azure-vars.env" diff --git a/.travis/setup-windows.sh b/.ci/setup-windows.sh old mode 100644 new mode 100755 similarity index 96% rename from .travis/setup-windows.sh rename to .ci/setup-windows.sh index 3054abbd36..f01d3afe95 --- a/.travis/setup-windows.sh +++ b/.ci/setup-windows.sh @@ -111,6 +111,6 @@ fi # BUILD is the name of the release artifact # AVVER is used for GitHub releases, it is the version number. BRANCH="${REPO_NAME}/${REPO_BRANCH}" -echo "BRANCH=$BRANCH" > .travis/azure-vars.env -echo "BUILD=$BUILD" >> .travis/azure-vars.env -echo "AVVER=$AVVER" >> .travis/azure-vars.env +echo "BRANCH=$BRANCH" > .ci/azure-vars.env +echo "BUILD=$BUILD" >> .ci/azure-vars.env +echo "AVVER=$AVVER" >> .ci/azure-vars.env diff --git a/.travis/travis.env b/.ci/travis.env similarity index 97% rename from .travis/travis.env rename to .ci/travis.env index d80dc2d3a5..6dad179fbf 100644 --- a/.travis/travis.env +++ b/.ci/travis.env @@ -3,6 +3,7 @@ TRAVIS_PULL_REQUEST TRAVIS_BRANCH TRAVIS_COMMIT # Variables set by Azure Pipelines +IS_AZURE BUILD_REASON BUILD_SOURCEVERSION BUILD_ARTIFACTSTAGINGDIRECTORY diff --git a/.travis.yml b/.travis.yml index 089af37a16..b44771be1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ jobs: cache: ccache compiler: gcc install: "docker pull rpcs3/rpcs3-travis-xenial:1.2" - script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash' + script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh' - os: linux dist: xenial env: @@ -21,7 +21,7 @@ jobs: cache: ccache compiler: clang install: "docker pull rpcs3/rpcs3-travis-xenial:1.2" - script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash' + script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh' # - os: osx # osx_image: xcode11.3 # addons: @@ -31,7 +31,7 @@ jobs: # - glew # - ninja # - qt - # script: "/bin/bash -ex .travis/build-mac.bash" + # script: "/bin/bash -ex .ci/build-mac.sh" # cache: ccache allow_failures: - os: osx diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1a43b3ccd7..1f417492a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,6 +20,7 @@ jobs: DEPLOY_APPIMAGE: true variables: CCACHE_DIR: $(Pipeline.Workspace)/ccache + IS_AZURE: true pool: vmImage: 'ubuntu-latest' steps: @@ -33,11 +34,11 @@ jobs: docker pull --quiet rpcs3/rpcs3-travis-xenial:1.2 docker run \ -v $(pwd):/rpcs3 \ - --env-file .travis/travis.env \ + --env-file .ci/travis.env \ -v $CCACHE_DIR:/root/.ccache \ -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/root/artifacts \ rpcs3/rpcs3-travis-xenial:1.2 \ - /bin/bash -ex /rpcs3/.travis/build-linux.bash + /rpcs3/.ci/build-linux.sh displayName: Docker setup and build - publish: $(Build.ArtifactStagingDirectory) @@ -64,10 +65,10 @@ jobs: path: $(CACHE_DIR) displayName: Cache - - bash: .travis/setup-windows.sh + - bash: .ci/setup-windows.sh displayName: Download and unpack dependencies - - bash: .travis/export-azure-vars.sh + - bash: .ci/export-azure-vars.sh displayName: Export Variables - task: VSBuild@1 @@ -78,7 +79,7 @@ jobs: configuration: 'Release - LLVM' displayName: Compile RPCS3 - - bash: .travis/deploy-windows.sh + - bash: .ci/deploy-windows.sh displayName: Pack up build artifacts - publish: $(Build.ArtifactStagingDirectory)