Unverified Commit 5d5557c7 authored by Denis_P's avatar Denis_P 🏑 Committed by GitHub
Browse files

Pre pop cargo home cache (#512)

* debug (CI): putting reports in artifacts [skip ci]

* debug (CI): remove unneded jobs, revert lated

* change (doc): fix artifacts

* change (CI): CARGO_HOME goes to branch level to avoid collapses; pre-population for it

* change (CI): small fix to codecov

* change (CI): DRY prepop script

* change (CI): DRY prepop script

* change (CI): docstring for pre-pop cache script

* change (CI): no cache for codecov report
parent 0b406924
Pipeline #110143 skipped
......@@ -15,7 +15,7 @@ variables:
GIT_STRATEGY: fetch
GIT_DEPTH: 100
CARGO_INCREMENTAL: 0
CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_JOB_NAME}"
CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
ALL_CRATES: "env storage storage/derive allocator prelude primitives lang lang/macro lang/ir"
......@@ -125,15 +125,18 @@ codecov:
# The `cargo-taurpalin` coverage reporting tool seems to have better code instrumentation and thus
# produces better results for Rust codebases in general. However, unlike `grcov` it requires
# running docker with `--security-opt seccomp=unconfined` which is why we use `grcov` instead.
# We removed the `-Cinline-threshold=0` flag from the above `RUSTFLAGS` since it was bugged
# at the time and lead to inlining of functions that shouldn't be inlined for the coverage
# report. (More information here: https://github.com/Kogia-sima/rust-covfix/issues/2)
script:
# RUSTFLAGS are the cause target cache can't be used here
before_script:
- cargo -vV
- rustc -vV
- rustup show
- bash --version
- unset "CARGO_TARGET_DIR"
- unset "CARGO_HOME"
- cargo clean
- cargo test --verbose --all-features --no-fail-fast --workspace
script:
# RUSTFLAGS are the cause target cache can't be used here
- cargo build --verbose --all-features --workspace
- cargo test --verbose --all-features --no-fail-fast --workspace
# coverage with branches
- grcov ./target -s . -t lcov --llvm --branch --ignore-not-existing --ignore "/*" --ignore "tests/*" -o lcov-w-branch.info
- rust-covfix -o lcov-w-branch.info lcov-w-branch.info
......
......@@ -5,16 +5,27 @@ set -u
# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
# create such directory and
# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists
if [[ ! -d $CARGO_TARGET_DIR ]]; then
mkdir -p "/ci-cache/$CI_PROJECT_NAME/targets/$CI_COMMIT_REF_NAME";
FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/targets" -mindepth 2 -maxdepth 2 \
-type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
if [[ -d $FRESH_CACHE ]]; then
echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
time cp -r "$FRESH_CACHE" "$CARGO_TARGET_DIR";
# caches are in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}
function prepopulate {
if [[ ! -d $1 ]]; then
mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME";
FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
-type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
if [[ -d $FRESH_CACHE ]]; then
echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
time cp -r "$FRESH_CACHE" "$1";
else
echo "_____No such $2 dir, proceeding from scratch_____";
fi
else
echo "_____No such targets dir, proceeding from scratch_____";
echo "____No need to prepopulate $2 cache____";
fi
else
echo "____No need to prepopulate CARGO_TARGET_DIR cache____";
fi
}
# CARGO_HOME cache was moved to the same "project/cache_type/branch_name/job_name" level as
# CARGO_TARGET_DIR because of frequent weird data-race issues. This just means that the same cache that
# would have been used for the entire pipeline will be duplicated for the each job.
prepopulate "$CARGO_HOME" cargo
prepopulate "$CARGO_TARGET_DIR" targets
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment