Check all crates (#12709)

* check all crates individually

It's relevant to check workspace crates individually because otherwise their compilation problems
due to feature misconfigurations won't be caught, as exemplified by
https://github.com/paritytech/substrate/issues/12705

* adapt to lack of multiple macos runners

https://github.com/paritytech/substrate/pull/12709#discussion_r1022868752

* fix cancel-pipeline-cargo-check-each-crate-macos

* fix cargo-check-each-crate-macos again

* time command execution

* fix YAML anchors

* add explanation for rounding division

* ensure the minimum of one crate per group

* collect artifacts for pipeline stopper

* revert hardcoded crates_per_group

* re-add crates_per_group=1

substrate/.gitlab-ci.yml

@@ -136,7 +136,7 @@ default:
 # exclude cargo-check-benches from such runs
 .test-refs-check-benches:
   rules:
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline"  && $CI_IMAGE =~ /staging$/ 
+    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline"  && $CI_IMAGE =~ /staging$/
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -329,10 +329,20 @@ cancel-pipeline-test-linux-stable-int:
   needs:
     - job:                         test-linux-stable-int
 
-cancel-pipeline-cargo-check-subkey:
+cancel-pipeline-cargo-check-each-crate-1:
   extends:                         .cancel-pipeline-template
   needs:
-    - job:                         cargo-check-subkey
+    - job:                         "cargo-check-each-crate 1/2"
+
+cancel-pipeline-cargo-check-each-crate-2:
+  extends:                         .cancel-pipeline-template
+  needs:
+    - job:                         "cargo-check-each-crate 2/2"
+
+cancel-pipeline-cargo-check-each-crate-macos:
+  extends:                         .cancel-pipeline-template
+  needs:
+    - job:                         cargo-check-each-crate-macos
 
 cancel-pipeline-check-tracing:
   extends:                         .cancel-pipeline-template

substrate/scripts/ci/gitlab/check-each-crate.sh

+#!/usr/bin/env bash
+
+## A script that checks each workspace crate individually.
+## It's relevant to check workspace crates individually because otherwise their compilation problems
+## due to feature misconfigurations won't be caught, as exemplified by
+## https://github.com/paritytech/substrate/issues/12705
+
+set -Eeu -o pipefail
+shopt -s inherit_errexit
+
+set -x
+
+target_group="$1"
+groups_total="$2"
+
+readarray -t workspace_crates < <(\
+  cargo tree --workspace --depth 0 | \
+  awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }'
+)
+
+crates_total=${#workspace_crates[*]}
+
+if [ "$crates_total" -lt "$groups_total" ]; then
+  # `crates_total / groups_total` would result in 0, so round it up to 1
+  crates_per_group=1
+else
+  # We add `crates_total % groups_total > 0` (which evaluates to 1 in case there's a remainder for
+  # `crates_total % groups_total`) to round UP `crates_total / groups_total` 's
+  # potentially-fractional result to the nearest integer. Doing that ensures that we'll not miss any
+  # crate in case `crates_total / groups_total` would normally result in a fractional number, since
+  # in those cases Bash would round DOWN the result to the nearest integer. For example, if
+  # `crates_total = 5` and `groups_total = 2`, then `crates_total / groups_total` would round down
+  # to 2; since the loop below would then step by 2, we'd miss the 5th crate.
+  crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) ))
+fi
+
+group=1
+for ((i=0; i < crates_total; i += crates_per_group)); do
+  if [ $group -eq "$target_group" ]; then
+    for crate in "${workspace_crates[@]:$i:$crates_per_group}"; do
+      cargo check --locked --release -p "$crate"
+    done
+    break
+  fi
+  group=$(( group + 1 ))
+done

substrate/scripts/ci/gitlab/pipeline/build.yml

@@ -90,9 +90,6 @@ build-linux-substrate:
   variables:
     # this variable gets overriden by "rusty-cachier environment inject", use the value as default
     CARGO_TARGET_DIR:              "$CI_PROJECT_DIR/target"
-  needs:
-    - job:                         cargo-check-subkey
-      artifacts:                   false
   before_script:
     - mkdir -p ./artifacts/subkey
     - !reference [.rusty-cachier, before_script]

substrate/scripts/ci/gitlab/pipeline/test.yml

@@ -133,24 +133,8 @@ node-bench-regression-guard:
        --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
   after_script: [""]
 
-cargo-check-subkey:
-  stage:                           test
-  extends:
-    - .docker-env
-    - .test-refs
-    - .pipeline-stopper-artifacts
-  script:
-    - rusty-cachier snapshot create
-    - cd ./bin/utils/subkey
-    - SKIP_WASM_BUILD=1 time cargo check --locked --release
-    - rusty-cachier cache upload
-
 cargo-check-try-runtime:
   stage:                           test
-  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
-  needs:
-    - job:                         cargo-check-subkey
-      artifacts:                   false
   extends:
     - .docker-env
     - .test-refs
@@ -393,6 +377,9 @@ test-full-crypto-feature:
 
 test-wasmer-sandbox:
   stage:                           test
+  needs:
+    - job:                         cargo-check-wasmer-sandbox
+      artifacts:                   false
   extends:
     - .docker-env
     - .test-refs-wasmer-sandbox
@@ -409,18 +396,6 @@ test-wasmer-sandbox:
     - time cargo nextest run --locked --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi
 
-cargo-check-macos:
-  stage:                           test
-  extends:                         .test-refs-no-trigger
-  before_script:
-    - !reference [.rust-info-script, script]
-  variables:
-    SKIP_WASM_BUILD:               1
-  script:
-    - time cargo check --locked --release
-  tags:
-    - osx
-
 check-rustdoc:
   stage:                           test
   variables:
@@ -435,3 +410,39 @@ check-rustdoc:
     - rusty-cachier snapshot create
     - time cargo +nightly doc --locked --workspace --all-features --verbose --no-deps
     - rusty-cachier cache upload
+
+cargo-check-each-crate:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+    - .collect-artifacts
+    - .pipeline-stopper-artifacts
+  variables:
+    # $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all
+    # "cargo-check-each-crate I/N" jobs
+    CI_JOB_NAME:                   cargo-check-each-crate
+  script:
+    - rusty-cachier snapshot create
+    - time ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
+    # need to update cache only from one job
+    - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi
+  parallel: 2
+
+cargo-check-each-crate-macos:
+  stage: test
+  extends:
+    - .test-refs
+    - .collect-artifacts
+    - .pipeline-stopper-artifacts
+  before_script:
+    - !reference [.rust-info-script, script]
+    - !reference [.pipeline-stopper-vars, script]
+  variables:
+    SKIP_WASM_BUILD:               1
+  script:
+    # TODO: enable rusty-cachier once it supports Mac
+    # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
+    - time ./scripts/ci/gitlab/check-each-crate.sh 1 1
+  tags:
+    - osx