Introduce `rusty-cachier` and remove old `pre_cache.sh` (#556)

.gitlab-ci.yml

@@ -9,16 +9,16 @@ stages:
   - check
   - test
   - build
+  - notify
 
 variables:
   GIT_STRATEGY:                    fetch
   GIT_DEPTH:                       "100"
-  CARGO_HOME:                      "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
-  CARGO_TARGET_DIR:                "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
   RUST_LIB_BACKTRACE:              "0"
   # this var is changed to "-:staging" when the CI image gets rebuilt
   # read more https://github.com/paritytech/cargo-contract/pull/115
   CI_IMAGE:                        "paritytech/contracts-ci-linux:production"
+  RUSTY_CACHIER_SINGLE_BRANCH:     master
 
 workflow:
   rules:
@@ -33,19 +33,32 @@ workflow:
     paths:
       - artifacts/
 
+.rusty-cachier:
+  before_script:
+    - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
+    - rusty-cachier environment check --gracefully
+    - $(rusty-cachier environment inject)
+    - rusty-cachier project mtime
+  after_script:
+    - rusty-cachier snapshot destroy
+
 .docker-env:                       &docker-env
-  image:                           "${CI_IMAGE}"
+  image:
+    name:                          "${CI_IMAGE}"
+    entrypoint:                    ['bash', '-c', 'exec unshare -m -r bash']
   before_script:
     - cargo -vV
     - rustc -vV
     - rustup show
     - cargo dylint --version
     - bash --version
-    - ./scripts/ci/pre_cache.sh
     # global RUSTFLAGS overrides the linker args so this way is better to pass the flags
     - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' > ${CARGO_HOME}/config
     - sccache -s
     - git show
+    - !reference [.rusty-cachier, before_script]
+  after_script:
+    - !reference [.rusty-cachier, after_script]
   rules:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -86,6 +99,7 @@ test-dylint:
   stage:                           test
   <<:                              *docker-env
   script:
+    - rusty-cachier snapshot create
     - cd ink_linting/
     - mv _Cargo.toml Cargo.toml
 
@@ -105,17 +119,21 @@ test-dylint:
     - unset RUSTC_WRAPPER
 
     - cargo test --verbose --all-features
+    - rusty-cachier cache upload
 
 test:
   stage:                           test
   <<:                              *docker-env
   script:
+    - rusty-cachier snapshot create
     - cargo test --verbose --workspace --all-features
+    - rusty-cachier cache upload
 
 test-new-project-template:
   stage:                           test
   <<:                              *docker-env
   script:
+    - rusty-cachier snapshot create
     - cargo run -- contract new new_project
 
     # needed because otherwise:
@@ -131,6 +149,7 @@ test-new-project-template:
     - cargo test --verbose --all
     - cargo fmt --verbose --all -- --check
     - cargo clippy --verbose --manifest-path Cargo.toml -- -D warnings;
+    - rusty-cachier cache upload
 
 # With the introduction of `ink_linting` in `build.rs` the installation process
 # is more elaborate now and as part of it the `ink_linting` crate is built locally.
@@ -140,6 +159,7 @@ test-registry-publish-install:
   stage:                           test
   <<:                              *docker-env
   before_script:
+    - !reference [.rusty-cachier, before_script]
     # Set up a local registry.
     - mkdir -p ./estuary/crates/ ./estuary/indices/
     - estuary --base-url=http://0.0.0.0:7878 --crate-dir ./estuary/crates/ --index-dir ./estuary/indices &
@@ -147,6 +167,7 @@ test-registry-publish-install:
     - echo -e '[registries]\nestuary = { index = "http://0.0.0.0:7878/git/index" }' > .cargo/config.toml
     - echo 0000 | cargo login --registry estuary
   script:
+    - rusty-cachier snapshot create
     - cargo publish --registry estuary
     - cargo install cargo-contract --index http://0.0.0.0:7878/git/index
 
@@ -154,6 +175,7 @@ test-registry-publish-install:
     - cargo run -- contract new new_project
     - echo "[workspace]" >> new_project/Cargo.toml
     - cargo run --all-features -- contract check --manifest-path new_project/Cargo.toml
+    - rusty-cachier cache upload
 
 #### stage:                        build (default features)
 
@@ -167,7 +189,18 @@ build:
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
+    - rusty-cachier snapshot create
     - cargo build --verbose --release
+    - rusty-cachier cache upload
   after_script:
+    - !reference [.rusty-cachier, after_script]
     - mkdir -p ./artifacts/cargo-contract/
     - cp ${CARGO_TARGET_DIR}/release/cargo-contract ./artifacts/cargo-contract/
+
+#### stage:                        notify (rusty-cachier notification about the latest revision)
+
+rusty-cachier-notify:
+  stage:                           notify
+  <<:                              *docker-env
+  script:
+    - rusty-cachier cache notify

scripts/ci/pre_cache.sh

-#!/bin/bash
-
-set -u
-
-# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
-# create such directory and
-# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists
-
-# caches are in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}
-
-function prepopulate {
-  if [[ ! -d $1 ]]; then
-    mkdir -p "$1";
-    FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
-      -type d -name "$CI_JOB_NAME" -not -path "$1" -exec stat --printf="%Y\t%n\n" {} \; \
-      |sort -n -r |head -1 |cut -f2);
-    if [[ -d "$FRESH_CACHE" ]]; then
-      echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
-      time cp -rf "$FRESH_CACHE" "$1";
-    else
-      echo "_____No such $2 dir, proceeding from scratch_____";
-    fi
-  else
-    echo "____No need to prepopulate $2 cache____";
-  fi
-}
-
-# CARGO_HOME cache was moved to the same "project/cache_type/branch_name/job_name" level as
-# CARGO_TARGET_DIR because of frequent weird data-race issues. This just means that the same cache that 
-# would have been used for the entire pipeline will be duplicated for the each job.
-prepopulate "$CARGO_HOME" cargo
-prepopulate "$CARGO_TARGET_DIR" targets