Simnet triggers and images (#2714)

* CI: new image for simnet

* CI: chore

* CI: separate image for collator

* dockerfile: it's bad upgrading in the images

* CI: correct links to Dockerfiles

* CI: right dotenv usage

* CI: workaround GitLab's bug that failed trigger status is not reported and 'job runs forever'

* CI: debug: need instead of rules to avoid the bug

* dockerfile: fix storage

* CI: workaround sending the status when the trigger is failed

* CI: fix needed job

* CI: edit comments

* CI: trigger simnet with API

* CI: triggering script

* CI: remove real-overseer

* CI: trigger real simnet

polkadot/.gitlab-ci.yml

@@ -38,7 +38,7 @@ default:
     when:                          on_success
     expire_in:                     28 days
     paths:
-      - artifacts/
+      - ./artifacts/
 
 .kubernetes-env:                   &kubernetes-env
   tags:
@@ -160,7 +160,7 @@ check-runtime-benchmarks:
         sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
       EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
     fi
-  - echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
+  - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
   - echo -n ${VERSION} > ./artifacts/VERSION
   - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
   - cp -r scripts/docker/* ./artifacts
@@ -170,22 +170,41 @@ build-linux-release:
   <<:                              *collect-artifacts
   <<:                              *docker-env
   <<:                              *compiler-info
-  variables:
-    EXTRA_FLAGS:                   ""
   rules:
-    # extra features when building on `rococo-v1` branch and manual build on PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      variables:
-        RUSTFLAGS:                 "-Cdebug-assertions=y"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
       variables:
-        RUSTFLAGS:                 "-Cdebug-assertions=y"
-    - when:                        always
+        RUSTFLAGS:                     "-Cdebug-assertions=y"
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+      variables:
+        RUSTFLAGS:                     "-Cdebug-assertions=y"
   script:
-    - time cargo build --release --verbose ${EXTRA_FLAGS}
+    - time cargo build --release --verbose
     - sccache -s
     - *pack-artifacts
 
+build-adder-collator:
+  stage:                           test
+  <<:                              *collect-artifacts
+  <<:                              *docker-env
+  <<:                              *compiler-info
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+  script:
+    - time cargo build --release --verbose -p test-parachain-adder-collator
+    - sccache -s
+    # pack artifacts
+    - mkdir -p ./artifacts
+    - mv ./target/release/adder-collator ./artifacts/.
+    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
+    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
+    - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
+    - cp -r scripts/docker/* ./artifacts
+
 #### stage:                        build
 
 check-transaction-versions:
@@ -213,12 +232,13 @@ generate-impl-guide:
   script:
     - mdbook build roadmap/implementers-guide
 
-.build-push-docker-image:          &build-push-docker-image
+.build-push-image:                 &build-push-image
   <<:                              *kubernetes-env
   image:                           quay.io/buildah/stable
   variables:
     GIT_STRATEGY:                  none
-    # DOCKERFILE:                  scripts/docker/Dockerfile
+    # scripts/docker/Dockerfile
+    DOCKERFILE:                    Dockerfile
     IMAGE_NAME:                    docker.io/parity/polkadot
     DOCKER_USER:                   ${Docker_Hub_User_Parity}
     DOCKER_PASS:                   ${Docker_Hub_Pass_Parity}
@@ -236,54 +256,79 @@ generate-impl-guide:
         --format=docker
         --build-arg VCS_REF="${CI_COMMIT_SHA}"
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_NAME}"
         --tag "$IMAGE_NAME:$VERSION"
-        --tag "$IMAGE_NAME:$EXTRATAG" .
+        --tag "$IMAGE_NAME:$EXTRATAG"
+        --file ${DOCKERFILE} .
     # The job will success only on the protected branch
     - echo "$DOCKER_PASS" |
         buildah login --username "$DOCKER_USER" --password-stdin docker.io
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
-    # pass artifacts to the trigget-simnet job
-    # this twist is to match the similar logic in substrate
-    - echo "IMAGE_NAME=${IMAGE_NAME}" > build.env
-    - echo "VERSION=${EXTRATAG}" >> build.env
   after_script:
     - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the trigger-simnet job
+    - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/build.env
+    - echo "IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/build.env
   artifacts:
     reports:
       # this artifact is used in trigger-simnet job
       # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-      dotenv: artifacts/build.env
-
+      dotenv: ./artifacts/build.env
 
-publish-dockerhub:
+publish-polkadot-image:
   stage:                           build
-  <<:                              *build-push-docker-image
+  <<:                              *build-push-image
   rules:
   # Don't run on releases - this is handled by the Github Action here:
   # .github/workflows/publish-docker-release.yml
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
       when: never
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+      variables:
+        IMAGE_NAME:                    docker.io/paritypr/synth-wave
+        DOCKER_USER:                   ${PARITYPR_USER}
+        DOCKER_PASS:                   ${PARITYPR_PASS}
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       variables:
         IMAGE_NAME:                docker.io/parity/rococo
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      variables:
-        # image to be tested with Simnet
-        IMAGE_NAME:                docker.io/paritypr/synth-wave
-        DOCKER_USER:               ${PARITYPR_USER}
-        DOCKER_PASS:               ${PARITYPR_PASS}
+  needs:
+    - job:                         build-linux-release
+      artifacts:                   true
+
+publish-adder-collator-image:
+  # service image for Simnet
+  stage:                           build
+  <<:                              *build-push-image
+  variables:
+    # scripts/docker/collator.Dockerfile
+    DOCKERFILE:                    collator.Dockerfile
+    IMAGE_NAME:                    docker.io/paritypr/colander
+    DOCKER_USER:                   ${PARITYPR_USER}
+    DOCKER_PASS:                   ${PARITYPR_PASS}
+  rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_COMMIT_REF_NAME == "master"
+    # FIXME: remove me after merging
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
   needs:
-    - job:                         build-linux-release
+    - job:                         build-adder-collator
       artifacts:                   true
+  after_script:
+    - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the trigger-simnet job
+    - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" > ./artifacts/collator.env
+  artifacts:
+    reports:
+      # this artifact is used in trigger-simnet job
+      dotenv: ./artifacts/collator.env
 
 #### stage:                        publish
 
-publish-s3-release:
+publish-s3-release:                &publish-s3
   stage:                           publish
   needs:
     - job:                         build-linux-release
@@ -295,10 +340,6 @@ publish-s3-release:
     BUCKET:                        "releases.parity.io"
     PREFIX:                        "polkadot/${ARCH}-${DOCKER_OS}"
   rules:
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      variables:
-        PREFIX:                    "rococo/${ARCH}-${DOCKER_OS}"
-    - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
@@ -327,6 +368,14 @@ publish-s3-release:
     - aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/
         --recursive --human-readable --summarize
 
+publish-s3-adder-collator:
+  <<:                              *publish-s3
+  variables:
+    PREFIX:                        "rococo/${ARCH}-${DOCKER_OS}"
+  needs:
+    - job:                         build-adder-collator
+      artifacts:                   true
+
 #### stage:                        deploy
 
 deploy-polkasync-kusama:
@@ -335,29 +384,36 @@ deploy-polkasync-kusama:
     # former .rules-build
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       when: never
-    - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   variables:
     POLKADOT_CI_COMMIT_NAME:       "${CI_COMMIT_REF_NAME}"
-    POLKADOT_CI_COMMIT_REF:        "${CI_COMMIT_REF}"
+    POLKADOT_CI_COMMIT_REF:        "${CI_COMMIT_SHORT_SHA}"
   allow_failure:                   true
   trigger:                         "parity/infrastructure/parity-testnet"
 
 trigger-simnet:
   stage:                           deploy
-  <<:                              *rules-test
+  image:                           paritytech/tools:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   needs:
-    - job:                         publish-dockerhub
-  # `build.env` is taken by the triggered job from `publish-dockerhub` job with an
-  # exact `$VERSION` (`$EXTRATAG` here, i.e. `2643-0.8.29-5f689e0a-6b24dc54`)
+    - job:                         publish-polkadot-image
+    - job:                         publish-adder-collator-image
+  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here, 
+  # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
+  # `collator.env` bears adder-collator unique build tag. In non-triggered builds it
+  # can be called by `master` tag.
   # Simnet uses an image published on PRs with this exact version for triggered runs
-  # on commits. And parity/rococo:rococo-v1 for runs not launched by this job.
-  trigger:
-    project:                       parity/simnet
-    branch:                        master
-    strategy:                      depend
+  # on commits. And parity/rococo:rococo-v1 for the runs not launched by this job.
   variables:
     TRGR_PROJECT:                  ${CI_PROJECT_NAME}
     TRGR_REF:                      ${CI_COMMIT_REF_NAME}
+    # simnet project ID
+    DWNSTRM_ID:                    332
+  script:
+    # API trigger for a simnet job
+    - ./scripts/gitlab/trigger_pipeline.sh

polkadot/roadmap/implementers-guide/src/SUMMARY.md

@@ -12,6 +12,7 @@
 - [Runtime Architecture](runtime/README.md)
   - [Initializer Module](runtime/initializer.md)
   - [Configuration Module](runtime/configuration.md)
+  - [Shared](runtime/shared.md)
   - [Disputes Module](runtime/disputes.md)
   - [Paras Module](runtime/paras.md)
   - [Scheduler Module](runtime/scheduler.md)

polkadot/roadmap/implementers-guide/src/runtime/initializer.md

@@ -30,7 +30,7 @@ The other parachains modules are initialized in this order:
 1. UMP
 1. HRMP
 
-The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared][shared.md] module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module.
+The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared](shared.md) module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module.
 
 Set `HasInitialized` to true.
 

polkadot/scripts/docker/Dockerfile

@@ -3,10 +3,11 @@ FROM debian:buster-slim
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME
 
 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/polkadot" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="polkadot: a platform for web3" \
 	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
@@ -18,7 +19,6 @@ ENV RUST_BACKTRACE 1
 
 # install tools and dependencies
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
 	DEBIAN_FRONTEND=noninteractive apt-get install -y \
 		libssl1.1 \
 		ca-certificates \
@@ -45,4 +45,3 @@ EXPOSE 30333 9933 9944
 VOLUME ["/polkadot"]
 
 ENTRYPOINT ["/usr/local/bin/polkadot"]
-

polkadot/scripts/docker/collator.Dockerfile

+FROM debian:buster-slim
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+ARG IMAGE_NAME
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="${IMAGE_NAME}" \
+	io.parity.image.description="polkadot: a platform for web3" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://github.com/paritytech/polkadot/"
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+
+# install tools and dependencies
+RUN apt-get update && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y \
+		libssl1.1 \
+		ca-certificates \
+		curl && \
+# apt cleanup
+	apt-get autoremove -y && \
+	apt-get clean && \
+	find /var/lib/apt/lists/ -type f -not -name lock -delete; \
+# add user and link ~/.local/share/adder-collator to /data
+	useradd -m -u 1000 -U -s /bin/sh -d /adder-collator adder-collator && \
+	mkdir -p /data /adder-collator/.local/share && \
+	chown -R adder-collator:adder-collator /data && \
+	ln -s /data /adder-collator/.local/share/polkadot
+
+# add adder-collator binary to docker image
+COPY ./adder-collator /usr/local/bin
+
+USER adder-collator
+
+# check if executable works in this container
+RUN /usr/local/bin/adder-collator --version
+
+EXPOSE 30333 9933 9944
+VOLUME ["/adder-collator"]
+
+ENTRYPOINT ["/usr/local/bin/adder-collator"]

polkadot/scripts/gitlab/trigger_pipeline.sh

+#!/bin/bash
+
+set -eu
+# API trigger another project's pipeline
+curl --silent \
+    -X POST \
+    -F "token=${CI_JOB_TOKEN}" \
+    -F "ref=master" \
+    -F "variables[TRGR_PROJECT]=${TRGR_PROJECT}" \
+    -F "variables[TRGR_REF]=${TRGR_REF}" \
+    -F "variables[IMAGE_NAME]=${IMAGE_NAME}" \
+    -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
+    -F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
+    "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
+        tee pipeline
+
+PIPELINE_ID=$(cat pipeline | jq ".id")
+echo "\nWaiting on ${PIPELINE_ID} status..."
+
+# This part polls for the triggered pipeline status, the native
+# `trigger` job does not return this status via API.
+# This is a workaround for a Gitlab bug, waits here until
+# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
+# The timeout is 360 curls with 8 sec interval, roughly an hour.
+
+function get_status() {
+    curl --silent \
+        --header "PRIVATE-TOKEN: ${PIPELINE_TOKEN}" \
+        "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/pipelines/${PIPELINE_ID}" | \
+            jq --raw-output ".status";
+}
+
+for i in $(seq 1 360); do
+    STATUS=$(get_status);
+    echo "Triggered pipeline status is ${STATUS}";
+    if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
+        echo "Busy...";
+    elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
+        exit 1;
+    elif [[ ${STATUS} =~ ^(success)$ ]]; then
+        exit 0;
+    else
+        exit 1;
+    fi
+sleep 8;
+done