From 4bd04392de94212f693334de8beb72201a948ac9 Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Thu, 12 May 2022 02:07:27 +0300 Subject: [PATCH 1/7] Introduce `rusty-cachier` and remove old `pre_cache.sh` --- .gitlab-ci.yml | 25 ++++++++++++++++++++++--- scripts/ci/pre_cache.sh | 32 -------------------------------- 2 files changed, 22 insertions(+), 35 deletions(-) delete mode 100755 scripts/ci/pre_cache.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e04655be..390441f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,6 @@ stages: variables: GIT_STRATEGY: fetch GIT_DEPTH: "100" - CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" - CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" RUST_LIB_BACKTRACE: "0" # this var is changed to "-:staging" when the CI image gets rebuilt # read more https://github.com/paritytech/cargo-contract/pull/115 @@ -33,6 +31,15 @@ workflow: paths: - artifacts/ +.rusty-cachier: + before_script: + - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash + - rusty-cachier environment check --gracefully + - $(rusty-cachier environment inject) + - rusty-cachier project mtime + after_script: + - rusty-cachier snapshot destroy + .docker-env: &docker-env image: "${CI_IMAGE}" before_script: @@ -41,11 +48,13 @@ workflow: - rustup show - cargo dylint --version - bash --version - - ./scripts/ci/pre_cache.sh # global RUSTFLAGS overrides the linker args so this way is better to pass the flags - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' > ${CARGO_HOME}/config - sccache -s - git show + - !reference [.rusty-cachier, before_script] + after_script: + - !reference [.rusty-cachier, after_script] rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" @@ -86,6 +95,7 @@ test-dylint: stage: test <<: *docker-env script: + - rusty-cachier snapshot create - cd ink_linting/ - mv _Cargo.toml Cargo.toml @@ -105,6 +115,7 @@ test-dylint: - unset RUSTC_WRAPPER - cargo test --verbose --all-features + - rusty-cachier cache upload test: stage: test @@ -116,6 +127,7 @@ test-new-project-template: stage: test <<: *docker-env script: + - rusty-cachier snapshot create - cargo run -- contract new new_project # needed because otherwise: @@ -131,6 +143,7 @@ test-new-project-template: - cargo test --verbose --all - cargo fmt --verbose --all -- --check - cargo clippy --verbose --manifest-path Cargo.toml -- -D warnings; + - rusty-cachier cache upload # With the introduction of `ink_linting` in `build.rs` the installation process # is more elaborate now and as part of it the `ink_linting` crate is built locally. @@ -140,6 +153,7 @@ test-registry-publish-install: stage: test <<: *docker-env before_script: + - !reference [.rusty-cachier, before_script] # Set up a local registry. - mkdir -p ./estuary/crates/ ./estuary/indices/ - estuary --base-url=http://0.0.0.0:7878 --crate-dir ./estuary/crates/ --index-dir ./estuary/indices & @@ -147,6 +161,7 @@ test-registry-publish-install: - echo -e '[registries]\nestuary = { index = "http://0.0.0.0:7878/git/index" }' > .cargo/config.toml - echo 0000 | cargo login --registry estuary script: + - rusty-cachier snapshot create - cargo publish --registry estuary - cargo install cargo-contract --index http://0.0.0.0:7878/git/index @@ -154,6 +169,7 @@ test-registry-publish-install: - cargo run -- contract new new_project - echo "[workspace]" >> new_project/Cargo.toml - cargo run --all-features -- contract check --manifest-path new_project/Cargo.toml + - rusty-cachier cache upload #### stage: build (default features) @@ -167,7 +183,10 @@ build: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: + - rusty-cachier snapshot create - cargo build --verbose --release + - rusty-cachier cache upload after_script: + - !reference [.rusty-cachier, after_script] - mkdir -p ./artifacts/cargo-contract/ - cp ${CARGO_TARGET_DIR}/release/cargo-contract ./artifacts/cargo-contract/ diff --git a/scripts/ci/pre_cache.sh b/scripts/ci/pre_cache.sh deleted file mode 100755 index d295f403..00000000 --- a/scripts/ci/pre_cache.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -u - -# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME -# create such directory and -# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists - -# caches are in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME} - -function prepopulate { - if [[ ! -d $1 ]]; then - mkdir -p "$1"; - FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \ - -type d -name "$CI_JOB_NAME" -not -path "$1" -exec stat --printf="%Y\t%n\n" {} \; \ - |sort -n -r |head -1 |cut -f2); - if [[ -d "$FRESH_CACHE" ]]; then - echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____"; - time cp -rf "$FRESH_CACHE" "$1"; - else - echo "_____No such $2 dir, proceeding from scratch_____"; - fi - else - echo "____No need to prepopulate $2 cache____"; - fi -} - -# CARGO_HOME cache was moved to the same "project/cache_type/branch_name/job_name" level as -# CARGO_TARGET_DIR because of frequent weird data-race issues. This just means that the same cache that -# would have been used for the entire pipeline will be duplicated for the each job. -prepopulate "$CARGO_HOME" cargo -prepopulate "$CARGO_TARGET_DIR" targets -- GitLab From a6f5780b525e4fd7d2f1fcd6d56a9a266899e02d Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Thu, 12 May 2022 02:36:27 +0300 Subject: [PATCH 2/7] Add `unshare` magic --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 390441f2..2348a6e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,9 @@ workflow: - rusty-cachier snapshot destroy .docker-env: &docker-env - image: "${CI_IMAGE}" + image: + name: "${CI_IMAGE}" + entrypoint: ['bash', '-c', 'exec unshare -m -r bash'] before_script: - cargo -vV - rustc -vV -- GitLab From a37513b1e039035c6dbe44b22bc91fa928e6029e Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Thu, 12 May 2022 03:03:22 +0300 Subject: [PATCH 3/7] Add revision notification --- .gitlab-ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2348a6e5..6c051c54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ stages: - check - test - build + - notify variables: GIT_STRATEGY: fetch @@ -192,3 +193,11 @@ build: - !reference [.rusty-cachier, after_script] - mkdir -p ./artifacts/cargo-contract/ - cp ${CARGO_TARGET_DIR}/release/cargo-contract ./artifacts/cargo-contract/ + +#### stage: notify (rusty-cachier notification about the latest revision) + +rusty-cachier-notify: + stage: notify + <<: *docker-env + script: + - rusty-cachier cache notify -- GitLab From 6ea46804280040a76631cbc9a82d5c9f2db285d4 Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Fri, 13 May 2022 15:40:36 +0300 Subject: [PATCH 4/7] Trigger CI --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c051c54..0c4ba776 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,6 @@ # # pipelines can be triggered manually in the web - stages: - check - test -- GitLab From 5a635a049f4668e60ebf9bac612ec18db5de7de7 Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Fri, 13 May 2022 18:40:03 +0300 Subject: [PATCH 5/7] Trigger CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c4ba776..6c051c54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ # # pipelines can be triggered manually in the web + stages: - check - test -- GitLab From 4a914bed817fc71b2b349e058658b680d75fc6a3 Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Fri, 13 May 2022 19:25:40 +0300 Subject: [PATCH 6/7] Instrument `test` job too --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c051c54..4e3cbb79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,7 +124,9 @@ test: stage: test <<: *docker-env script: + - rusty-cachier snapshot create - cargo test --verbose --workspace --all-features + - rusty-cachier cache upload test-new-project-template: stage: test -- GitLab From cb930661e018a59adc20e81f350842a62d084cba Mon Sep 17 00:00:00 2001 From: Vladimir Istyufeev Date: Fri, 13 May 2022 19:34:50 +0300 Subject: [PATCH 7/7] Enable single branch mode --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e3cbb79..eb756b32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ variables: # this var is changed to "-:staging" when the CI image gets rebuilt # read more https://github.com/paritytech/cargo-contract/pull/115 CI_IMAGE: "paritytech/contracts-ci-linux:production" + RUSTY_CACHIER_SINGLE_BRANCH: master workflow: rules: -- GitLab