From 024b1840086199c9eb20d62c6c5c80a075f47838 Mon Sep 17 00:00:00 2001 From: xddxd Date: Sun, 4 Apr 2021 11:34:12 +0300 Subject: [PATCH] Cirrus build --- .ci/build-linux.sh | 10 ++-- .ci/deploy-linux.sh | 9 ++-- .ci/deploy-windows.sh | 6 +-- .ci/docker.env | 5 +- .ci/export-cirrus-vars.sh | 13 ++++++ .ci/setup-windows.sh | 2 +- .cirrus.yml | 96 +++++++++++++++++++++++++++++++++++++++ azure-pipelines.yml | 12 +++-- 8 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 .ci/export-cirrus-vars.sh diff --git a/.ci/build-linux.sh b/.ci/build-linux.sh index 5c71fc90b0..fe53dcfd2f 100755 --- a/.ci/build-linux.sh +++ b/.ci/build-linux.sh @@ -5,7 +5,9 @@ export QT_BASE_DIR=/opt/qt${QTVERMIN} export PATH=$QT_BASE_DIR/bin:$PATH export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib -cd rpcs3 || exit 1 +if [ -z "$CIRRUS_CI" ]; then + cd rpcs3 || exit 1 +fi # Pull all the submodules except llvm, since it is built separately and we just download that build # Note: Tried to use git submodule status, but it takes over 20 seconds @@ -57,9 +59,9 @@ cd .. shellcheck .ci/*.sh -# If it compiled succesfully let's deploy depending on the build pipeline (Azure Pipelines). -# Azure publishes PRs as artifacts only. -{ [ "$IS_AZURE" = "true" ]; +# If it compiled succesfully let's deploy. +# Azure and Cirrus publish PRs as artifacts only. +{ [ "$CI_HAS_ARTIFACTS" = "true" ]; } && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false" if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then diff --git a/.ci/deploy-linux.sh b/.ci/deploy-linux.sh index 2bde0370f7..2331dee004 100755 --- a/.ci/deploy-linux.sh +++ b/.ci/deploy-linux.sh @@ -33,7 +33,8 @@ if [ "$DEPLOY_APPIMAGE" = "true" ]; then printf "#include \nint main(){std::make_exception_ptr(0);std::pmr::get_default_resource();}" | $CXX -x c++ -std=c++2a -o ./appdir/usr/optional/checker - # Package it up and send it off - ./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir + ./squashfs-root/usr/bin/appimagetool "$APPDIR" + ls COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')" @@ -43,14 +44,14 @@ if [ "$DEPLOY_APPIMAGE" = "true" ]; then mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE" - # If we're building using Azure Pipelines, let's copy over the AppImage artifact + # If we're building using a CI, let's copy over the AppImage artifact if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then - cp "$RPCS3_APPIMAGE" ~/artifacts + cp "$RPCS3_APPIMAGE" "$ARTDIR" fi FILESIZE=$(stat -c %s ./rpcs3*.AppImage) SHA256SUM=$(sha256sum ./rpcs3*.AppImage | awk '{ print $1 }') - echo "${SHA256SUM};${FILESIZE}B" > /rpcs3/GitHubReleaseMessage.txt + echo "${SHA256SUM};${FILESIZE}B" > "$RELEASE_MESSAGE" fi diff --git a/.ci/deploy-windows.sh b/.ci/deploy-windows.sh index c612701e5f..f269fe45d4 100755 --- a/.ci/deploy-windows.sh +++ b/.ci/deploy-windows.sh @@ -4,7 +4,7 @@ ARTIFACT_DIR="$BUILD_ARTIFACTSTAGINGDIRECTORY" # Remove unecessary files -rm -f ./bin/rpcs3.exp ./bin/rpcs3.lib ./bin/rpcs3.pdb +rm -f ./bin/rpcs3.exp ./bin/rpcs3.lib ./bin/rpcs3.pdb ./bin/vc_redist.x64.exe # Prepare compatibility database for packaging, as well as # certificate for ssl (auto-updater) @@ -20,5 +20,5 @@ sha256sum "$BUILD" | awk '{ print $1 }' | tee "$BUILD.sha256" echo "$(cat "$BUILD.sha256");$(stat -c %s "$BUILD")B" > GitHubReleaseMessage.txt # Move files to publishing directory -mv -- "$BUILD" "$ARTIFACT_DIR" -mv -- "$BUILD.sha256" "$ARTIFACT_DIR" +cp -- "$BUILD" "$ARTIFACT_DIR" +cp -- "$BUILD.sha256" "$ARTIFACT_DIR" diff --git a/.ci/docker.env b/.ci/docker.env index b67a8646c3..2b36fb34c0 100644 --- a/.ci/docker.env +++ b/.ci/docker.env @@ -1,10 +1,13 @@ # Variables set by Azure Pipelines -IS_AZURE +CI_HAS_ARTIFACTS BUILD_REASON BUILD_SOURCEVERSION BUILD_ARTIFACTSTAGINGDIRECTORY BUILD_REPOSITORY_NAME BUILD_SOURCEBRANCHNAME +APPDIR +ARTDIR +RELEASE_MESSAGE # Variables for build matrix COMPILER DEPLOY_APPIMAGE diff --git a/.ci/export-cirrus-vars.sh b/.ci/export-cirrus-vars.sh new file mode 100644 index 0000000000..6553651d7b --- /dev/null +++ b/.ci/export-cirrus-vars.sh @@ -0,0 +1,13 @@ +#!/bin/sh -e + +# Export variables for later stages of the Azure pipeline +# Values done in this manner will appear as environment variables +# in later stages. + +# From pure-sh-bible +# Setting 'IFS' tells 'read' where to split the string. +while IFS='=' read -r key val; do + # Skip over lines containing comments. + [ "${key##\#*}" ] || continue + export "$key"="$val" +done < ".ci/azure-vars.env" diff --git a/.ci/setup-windows.sh b/.ci/setup-windows.sh index 4a000fec39..f02e965253 100755 --- a/.ci/setup-windows.sh +++ b/.ci/setup-windows.sh @@ -36,7 +36,7 @@ DEP_URLS=" \ # Pull all the submodules except llvm, since it is built separately and we just download that build # Note: Tried to use git submodule status, but it takes over 20 seconds # shellcheck disable=SC2046 -git submodule -q update --init --depth 1 $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules) +git submodule -q update --init --depth=1 --jobs=8 $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules) # Git bash doesn't have rev, so here it is rev() diff --git a/.cirrus.yml b/.cirrus.yml index eb52a41a0f..888a240a37 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,5 +1,101 @@ env: CIRRUS_CLONE_DEPTH: 0 # Unshallow clone to obtain proper GIT_VERSION + BUILD_REPOSITORY_NAME: $CIRRUS_REPO_FULL_NAME + SYSTEM_PULLREQUEST_SOURCEBRANCH: $CIRRUS_BRANCH + SYSTEM_PULLREQUEST_PULLREQUESTID: $CIRRUS_PR + BUILD_SOURCEVERSION: $CIRRUS_CHANGE_IN_REPO + RPCS3_TOKEN: ENCRYPTED[13a0f18de9285e6c880e3aecbb54258245a74872af7d0fcb92447fa1200ceb6a4dc8acc880ddbf5e653d46336563ca72] # Replace this when we're using Cirrus for master releases!! + +windows_task: + matrix: + - name: Cirrus Windows + windows_container: + image: cirrusci/windowsservercore:visualstudio2019 + cpu: 8 + memory: 16G + env: + CIRRUS_SHELL: "bash" + COMPILER: msvc + QT_VER_MAIN: '5' + BUILD_ARTIFACTSTAGINGDIRECTORY: ${CIRRUS_WORKING_DIR}\artifacts\ + QT_VER: '5.15.2' + QT_VER_MSVC: 'msvc2019' + QT_DATE: '202011130602' + QTDIR: C:\Qt\${QT_VER}\${QT_VER_MSVC}_64 + VULKAN_VER: '1.2.154.1' + VULKAN_SDK_SHA: 'b64471f3a720e649c1fae6535ea83b8c642655ebed1485bfdf15bf4d88f746d9' + VULKAN_SDK: C:\VulkanSDK\${VULKAN_VER} + CACHE_DIR: "./cache" + UPLOAD_COMMIT_HASH: 7d09e3be30805911226241afbb14f8cdc2eb054e + UPLOAD_REPO_FULL_NAME: "rpcs3/rpcs3-binaries-win" + deps_cache: + folder: "./cache" + #obj_cache: + # folder: "./tmp" + #obj2_cache: + # folder: "./rpcs3/x64" + setup_script: + - './.ci/get_keys-windows.sh' + - './.ci/setup-windows.sh' +# - choco install -y python # Needed for SPIRV, use either this or make a new Docker image +# spirv_script: +# - export PATH=${PATH}:"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin" +# - cd "${CIRRUS_WORKING_DIR}/Vulkan/spirv-tools-build" +# - msbuild.exe spirv-tools-build.vcxproj //p:Configuration=Release //m + rpcs3_script: + - export PATH=${PATH}:"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin" + - msbuild.exe rpcs3.sln //p:Configuration=Release //m + deploy_script: + - mkdir artifacts + - source './.ci/export-cirrus-vars.sh' + - './.ci/deploy-windows.sh' + artifacts: + name: Artifact + path: "*.7z*" +# push_script: | +# if [ "$CIRRUS_REPO_OWNER" = "RPCS3" ] && [ -z "$CIRRUS_PR" ] && [ "$CIRRUS_BRANCH" = "master" ]; then +# source './.ci/export-cirrus-vars.sh' +# './.ci/github-upload.sh' +# fi; + +linux_task: + container: + image: rpcs3/rpcs3-travis-xenial:1.7 + cpu: 4 + memory: 16G + env: + BUILD_ARTIFACTSTAGINGDIRECTORY: ${CIRRUS_WORKING_DIR}/artifacts + CCACHE_DIR: "/tmp/ccache_dir" + CCACHE_MAXSIZE: 300M + CI_HAS_ARTIFACTS: true + UPLOAD_COMMIT_HASH: d812f1254a1157c80fd402f94446310560f54e5f + UPLOAD_REPO_FULL_NAME: "rpcs3/rpcs3-binaries-linux" + DEPLOY_APPIMAGE: true + APPDIR: "./appdir" + ARTDIR: ${CIRRUS_WORKING_DIR}/artifacts/ + RELEASE_MESSAGE: "../GitHubReleaseMessage.txt" + ccache_cache: + folder: "/tmp/ccache_dir" + matrix: + - name: Cirrus Linux GCC + env: + COMPILER: gcc + gcc_script: + - mkdir artifacts + - ".ci/build-linux.sh" + - name: Cirrus Linux Clang + env: + COMPILER: clang + clang_script: + - mkdir artifacts + - ".ci/build-linux.sh" + artifacts: + name: Artifact + path: "artifacts/*" +# push_script: | +# if [ "$CIRRUS_REPO_OWNER" = "RPCS3" ] && [ -z "$CIRRUS_PR" ] && [ "$CIRRUS_BRANCH" = "master" ] && [ "$COMPILER" = "gcc" ]; then +# '.ci/github-upload.sh' +# fi; freebsd_task: matrix: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f6d29f187b..8c79542ba3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,10 +5,7 @@ trigger: tags: exclude: - '*' -pr: - branches: - include: - - master +pr: none jobs: - job: Linux_Build strategy: @@ -19,12 +16,17 @@ jobs: COMPILER: gcc variables: CCACHE_DIR: $(Pipeline.Workspace)/ccache - IS_AZURE: true + CI_HAS_ARTIFACTS: true UPLOAD_COMMIT_HASH: d812f1254a1157c80fd402f94446310560f54e5f UPLOAD_REPO_FULL_NAME: "RPCS3/rpcs3-binaries-linux" DEPLOY_APPIMAGE: true + APPDIR: "/rpcs3/build/appdir" + ARTDIR: "/root/artifacts" + RELEASE_MESSAGE: "/rpcs3/GitHubReleaseMessage.txt" + pool: vmImage: 'ubuntu-latest' + steps: - task: Cache@2 inputs: