From 8a4374a8a9ba62b4904c1a7777c0716d2f087ce9 Mon Sep 17 00:00:00 2001 From: Denis P Date: Mon, 26 Oct 2020 19:07:53 +0100 Subject: [PATCH 1/6] change (CI): some modern CI features --- .gitlab-ci.yml | 37 ++++++++++++++++++++----------------- scripts/pre_cache.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 scripts/pre_cache.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a957d10..fea6b918 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,13 +12,15 @@ stages: variables: GIT_STRATEGY: fetch - GIT_DEPTH: 3 - CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_JOB_NAME}" + GIT_DEPTH: "100" + CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" - CI_SERVER_NAME: "GitLab CI" - REGISTRY: "paritytech" - RUSTUP_TOOLCHAIN: nightly - RUST_LIB_BACKTRACE: 0 + RUST_LIB_BACKTRACE: "0" + +workflow: + rules: + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH .collect-artifacts: &collect-artifacts artifacts: @@ -29,28 +31,29 @@ variables: - artifacts/ .docker-env: &docker-env - image: ${REGISTRY}/contracts-ci-linux:latest + image: paritytech/contracts-ci-linux:latest before_script: - cargo -vV - rustc -vV - rustup show - bash --version - - mkdir -p ${CARGO_HOME}; touch ${CARGO_HOME}/config - - mkdir -p ${CARGO_TARGET_DIR} + - ./scripts/pre_cache.sh # global RUSTFLAGS overrides the linker args so this way is better to pass the flags - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' | tee ${CARGO_HOME}/config - sccache -s - git show - only: - - master - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - - schedules - - web - - /^[0-9]+$/ # PRs + rules: + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "tags" + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 dependencies: [] interruptible: true retry: - max: 2 + max: 2 when: - runner_system_failure - unknown_failure @@ -74,7 +77,7 @@ test: stage: test <<: *docker-env script: - - cargo test --verbose --all --all-features + - cargo test --verbose --workspace --all-features #### stage: build (default features) diff --git a/scripts/pre_cache.sh b/scripts/pre_cache.sh new file mode 100644 index 00000000..b6074a5c --- /dev/null +++ b/scripts/pre_cache.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -u + +# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME +# create such directory and +# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists + +# caches are in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME} + +function prepopulate { + if [[ ! -d $1 ]]; then + mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME"; + FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \ + -type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2); + if [[ -d $FRESH_CACHE ]]; then + echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____"; + time cp -r "$FRESH_CACHE" "$1"; + else + echo "_____No such $2 dir, proceeding from scratch_____"; + fi + else + echo "____No need to prepopulate $2 cache____"; + fi +} + +# CARGO_HOME cache was moved to the same "project/cache_type/branch_name/job_name" level as +# CARGO_TARGET_DIR because of frequent weird data-race issues. This just means that the same cache that +# would have been used for the entire pipeline will be duplicated for the each job. +prepopulate "$CARGO_HOME" cargo +prepopulate "$CARGO_TARGET_DIR" targets -- GitLab From 169d7c9477be8344276975b3462779da7ed69b70 Mon Sep 17 00:00:00 2001 From: Denis P Date: Mon, 26 Oct 2020 19:29:02 +0100 Subject: [PATCH 2/6] change (conf): chown --- scripts/pre_cache.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/pre_cache.sh diff --git a/scripts/pre_cache.sh b/scripts/pre_cache.sh old mode 100644 new mode 100755 -- GitLab From bd137833e7660ed6f3ad1eed3a3a9e90097ebd09 Mon Sep 17 00:00:00 2001 From: Denis P Date: Mon, 26 Oct 2020 19:52:14 +0100 Subject: [PATCH 3/6] change (CI): missed one `only` --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fea6b918..31b2c7c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,9 +85,11 @@ build: stage: build <<: *docker-env <<: *collect-artifacts - only: - - schedules - - master + rules: + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - cargo build --verbose --release after_script: -- GitLab From 8e8013de8a2b2d16eb6f8a11f0173a2aaf5a4f29 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 27 Oct 2020 11:13:04 +0100 Subject: [PATCH 4/6] change (CI): create config file in new CARGO_HOME --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31b2c7c1..67a81354 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,7 @@ workflow: - bash --version - ./scripts/pre_cache.sh # global RUSTFLAGS overrides the linker args so this way is better to pass the flags + - touch ${CARGO_HOME}/config - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' | tee ${CARGO_HOME}/config - sccache -s - git show -- GitLab From f4cb2b0623617c6a8cce38c741403139ea1c602f Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 27 Oct 2020 11:24:49 +0100 Subject: [PATCH 5/6] change (CI): mkdir for prepopulation 1 step deeper --- .gitlab-ci.yml | 3 +-- scripts/pre_cache.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67a81354..33edc5fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,8 +39,7 @@ workflow: - bash --version - ./scripts/pre_cache.sh # global RUSTFLAGS overrides the linker args so this way is better to pass the flags - - touch ${CARGO_HOME}/config - - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' | tee ${CARGO_HOME}/config + - printf '[build]\nrustflags = ["-C", "link-dead-code"]\n' > ${CARGO_HOME}/config - sccache -s - git show rules: diff --git a/scripts/pre_cache.sh b/scripts/pre_cache.sh index b6074a5c..3fc43937 100755 --- a/scripts/pre_cache.sh +++ b/scripts/pre_cache.sh @@ -10,7 +10,7 @@ set -u function prepopulate { if [[ ! -d $1 ]]; then - mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME"; + mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME/$CI_JOB_NAME"; FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \ -type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2); if [[ -d $FRESH_CACHE ]]; then -- GitLab From 281b851cc38be18275080a0c530252ae77ca92f6 Mon Sep 17 00:00:00 2001 From: Denis P Date: Tue, 27 Oct 2020 11:43:14 +0100 Subject: [PATCH 6/6] change (CI): do not look in the same dir --- scripts/pre_cache.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pre_cache.sh b/scripts/pre_cache.sh index 3fc43937..76723c2f 100755 --- a/scripts/pre_cache.sh +++ b/scripts/pre_cache.sh @@ -10,10 +10,10 @@ set -u function prepopulate { if [[ ! -d $1 ]]; then - mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME/$CI_JOB_NAME"; + mkdir -p "$1"; FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \ - -type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2); - if [[ -d $FRESH_CACHE ]]; then + -type d -name "$CI_JOB_NAME" -not -path "$1" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2); + if [[ -d "$FRESH_CACHE" ]]; then echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____"; time cp -r "$FRESH_CACHE" "$1"; else -- GitLab