From edd95b3749754d2ed0c5738588e872c87be91624 Mon Sep 17 00:00:00 2001
From: Evgeny Snitko <evgeny@parity.io>
Date: Fri, 9 Feb 2024 15:40:19 +0300
Subject: [PATCH] Add forklift to remaining jobs (#3236)
Add [forklift
caching](https://gitlab.parity.io/parity/infrastructure/ci_cd/forklift/forklift)
to remainig jobs
by .sh and .py scripts:
- cargo-check-each-crate x6 (`.gitlab/check-each-crate.py`)
- build-linux-stable (`polkadot/scripts/build-only-wasm.sh`)
by before_script:
- build-linux-substrate
- build-subkey-linux (with `.build-subkey` job)
- cargo-check-benches x2
**To disable feature set FORKLIFT_BYPASS variable to true in [project
settings in
gitlab](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/settings/ci_cd)**
(forklift now handles FORKLIFT_BYPASS by itself)
---
.gitlab-ci.yml | 8 ++------
.gitlab/check-each-crate.py | 2 +-
.gitlab/pipeline/build.yml | 6 ++++--
.gitlab/pipeline/test.yml | 1 +
polkadot/scripts/build-only-wasm.sh | 12 ++++++++++--
5 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c90a6c8e6e2..7f8796ca512 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,9 +121,8 @@ default:
before_script:
- 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
- chmod +x forklift
- - mkdir .forklift
- - cp $FL_FORKLIFT_CONFIG .forklift/config.toml
- - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*}
+ - mkdir ~/.forklift
+ - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
- shopt -s expand_aliases
- export PATH=$PATH:$(pwd)
- |
@@ -131,11 +130,8 @@ default:
echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
alias cargo="forklift cargo"
fi
- - ls -al
- - rm -f forklift.sock
#
- echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
- - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX"
.common-refs:
rules:
diff --git a/.gitlab/check-each-crate.py b/.gitlab/check-each-crate.py
index da2eaad36c5..9b654f8071a 100755
--- a/.gitlab/check-each-crate.py
+++ b/.gitlab/check-each-crate.py
@@ -55,7 +55,7 @@ for i in range(0, crates_per_group + overflow_crates):
print(f"Checking {crates[crate][0]}", file=sys.stderr)
- res = subprocess.run(["cargo", "check", "--locked"], cwd = crates[crate][1])
+ res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1])
if res.returncode != 0:
sys.exit(1)
diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml
index d9c67af7efd..482fc222aa0 100644
--- a/.gitlab/pipeline/build.yml
+++ b/.gitlab/pipeline/build.yml
@@ -314,8 +314,9 @@ build-linux-substrate:
# tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
# see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
- git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+ - !reference [.forklift-cache, before_script]
script:
- - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p staging-node-cli
+ - time WASM_BUILD_NO_COLOR=1 cargo build --locked --release -p staging-node-cli
- mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
- echo -n "Substrate version = "
- if [ "${CI_COMMIT_TAG}" ]; then
@@ -352,9 +353,10 @@ build-runtimes-polkavm:
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
before_script:
- mkdir -p ./artifacts/subkey
+ - !reference [.forklift-cache, before_script]
script:
- cd ./substrate/bin/utils/subkey
- - SKIP_WASM_BUILD=1 time cargo build --locked --release
+ - time SKIP_WASM_BUILD=1 cargo build --locked --release
# - cd -
# - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
# - echo -n "Subkey version = "
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
index e75700ffddc..b5e26d19489 100644
--- a/.gitlab/pipeline/test.yml
+++ b/.gitlab/pipeline/test.yml
@@ -224,6 +224,7 @@ cargo-check-benches:
git merge --verbose --no-edit FETCH_HEAD;
fi
fi'
+ - !reference [.forklift-cache, before_script]
parallel: 2
script:
- mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
diff --git a/polkadot/scripts/build-only-wasm.sh b/polkadot/scripts/build-only-wasm.sh
index b6da3319c82..50b786dab41 100755
--- a/polkadot/scripts/build-only-wasm.sh
+++ b/polkadot/scripts/build-only-wasm.sh
@@ -13,6 +13,14 @@ fi
WASM_BUILDER_RUNNER="$PROJECT_ROOT/target/release/wbuild-runner/$1"
+fl_cargo () {
+ if command -v forklift >/dev/null 2>&1; then
+ forklift cargo "$@";
+ else
+ cargo "$@";
+ fi
+}
+
if [ -z "$2" ]; then
export WASM_TARGET_DIRECTORY=$(pwd)
else
@@ -22,8 +30,8 @@ fi
if [ -d $WASM_BUILDER_RUNNER ]; then
export DEBUG=false
export OUT_DIR="$PROJECT_ROOT/target/release/build"
- cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \
+ fl_cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \
| grep -vE "cargo:rerun-if-|Executing build command"
else
- cargo build --release -p $1
+ fl_cargo build --release -p $1
fi
--
GitLab