Avoid unintentionally canceling the scheduled crate publishing job (#13088)

* avoid unintentionally canceling the scheduled crate publishing job

because publish-crates and publish-crates-manual share the resource group
"crates-publishing", any instance of publish-crates-manual cancels a running
instance of publish-crates, as demonstrated by
https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2212179

a workaround for that unintended interaction is to avoid creating instances of
publish-crates-manual and instead require pipelines to be triggered manually by
checking $CI_JOB_MANUAL == "true"

* check manual pipelines by $CI_PIPELINE_SOURCE instead of $CI_JOB_MANUAL

* make crate-publishing pipelines uninterruptible

* use conditional includes to work around interruptible limitations

* organize comments

* remove interruptible from common pipeline

* wip: check include

* wip: check include

* fix include

* fix include

* fix include

* fix yaml

* fix yaml

* remove shared common-pipeline

* wip: retry common-pipeline

* move .default-template to .gitlab-ci.yml

* fix the pipeline

add comments

* fix default-pipeline.yml

* revert publish-crates-manual to when: manual

* move "needs:" back to publish-crates

* avoid manual repetition

* improve previous commit

* try to avoid manual repetition

* fix indentation

* minor adjustments

* move defaults to top of .gitlab-ci.yml

* fix positioning on default in the diff

* comments

* indentation

* Apply suggestions from code review

Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>

Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>

substrate/.gitlab-ci.yml

@@ -57,16 +57,30 @@ variables:
   NEXTEST_SUCCESS_OUTPUT:          final
   ZOMBIENET_IMAGE:                 "docker.io/paritytech/zombienet:v1.3.22"
 
-default:
+.shared-default:                   &shared-default
   retry:
     max: 2
     when:
       - runner_system_failure
       - unknown_failure
       - api_failure
-  interruptible:                   true
   cache:                           {}
 
+.default-pipeline-definitions:
+  default:
+    <<:                            *shared-default
+    interruptible:                 true
+
+.crate-publishing-pipeline-definitions:
+  default:
+    <<:                            *shared-default
+    # The crate-publishing pipeline defaults to `interruptible: false` so that we'll be able to
+    # reach and run the publishing jobs despite the "Auto-cancel redundant pipelines" CI setting.
+    # The setting is relevant because the crate-publishing pipeline runs on `master`, thus future
+    # pipelines on `master` (e.g. created for new commits or other schedules) might unintendedly
+    # cancel the publishing jobs or its dependencies before we get to actually publish the crates.
+    interruptible:                 false
+
 .collect-artifacts:
   artifacts:
     name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
@@ -215,9 +229,15 @@ default:
     # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
 
+.crate-publishing-pipeline:
+  rules:
+    - if: $CI_COMMIT_REF_NAME != "master"
+      when: never
+
 .scheduled-crate-publishing-pipeline:
   rules:
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing"
+    - !reference [.crate-publishing-pipeline, rules]
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing"
 
 .crates-publishing-template:
   stage:                           test
@@ -265,6 +285,22 @@ include:
   - scripts/ci/gitlab/pipeline/publish.yml
   # zombienet jobs
   - scripts/ci/gitlab/pipeline/zombienet.yml
+  # The crate-publishing pipeline requires a customized `interruptible` configuration. Unfortunately
+  # `interruptible` can't currently be dynamically set based on variables as per:
+  # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349
+  # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023
+  # Thus we work around that limitation by using conditional includes.
+  # For crate-publishing pipelines: run it with defaults + `interruptible: false`. The WHOLE
+  # pipeline is made uninterruptible to ensure that test jobs also get a chance to run to
+  # completion, because the publishing jobs depends on them AS INTENDED: crates should not be
+  # published before their source code is checked.
+  - local: scripts/ci/gitlab/crate-publishing-pipeline.yml
+    rules:
+      - if: $PIPELINE == "automatic-crate-publishing"
+  # For normal pipelines: run it with defaults + `interruptible: true`
+  - local: scripts/ci/gitlab/default-pipeline.yml
+    rules:
+      - if: $PIPELINE != "automatic-crate-publishing"
 
 #### stage:                        deploy
 

substrate/scripts/ci/gitlab/crate-publishing-pipeline.yml

+default: !reference [.crate-publishing-pipeline-definitions, default]

substrate/scripts/ci/gitlab/default-pipeline.yml

+default: !reference [.default-pipeline-definitions, default]

substrate/scripts/ci/gitlab/pipeline/publish.yml

@@ -214,7 +214,7 @@ update-node-template:
   timeout:                         9h
   # A custom publishing environment is used for us to be able to set up protected secrets
   # specifically for it
-  environment: publish-crates
+  environment:                     publish-crates
   script:
     - rusty-cachier snapshot create
     - git clone
@@ -234,5 +234,6 @@ publish-crates:
 
 publish-crates-manual:
   extends:                         .publish-crates-template
+  rules:                           !reference [.crate-publishing-pipeline, rules]
   when:                            manual
   allow_failure:                   true