Unverified Commit 2daf65c7 authored by Denis_P's avatar Denis_P 🏑 Committed by GitHub
Browse files

Simnet triggers and images (#2714)

* CI: new image for simnet

* CI: chore

* CI: separate image for collator

* dockerfile: it's bad upgrading in the images

* CI: correct links to Dockerfiles

* CI: right dotenv usage

* CI: workaround GitLab's bug that failed trigger status is not reported and 'job runs forever'

* CI: debug: need instead of rules to avoid the bug

* dockerfile: fix storage

* CI: workaround sending the status when the trigger is failed

* CI: fix needed job

* CI: edit comments

* CI: trigger simnet with API

* CI: triggering script

* CI: remove real-overseer

* CI: trigger real simnet
parent fe8d808c
Pipeline #134495 failed with stages
in 22 minutes
......@@ -38,7 +38,7 @@ default:
when: on_success
expire_in: 28 days
paths:
- artifacts/
- ./artifacts/
.kubernetes-env: &kubernetes-env
tags:
......@@ -160,7 +160,7 @@ check-runtime-benchmarks:
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
fi
- echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
- cp -r scripts/docker/* ./artifacts
......@@ -170,22 +170,41 @@ build-linux-release:
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
variables:
EXTRA_FLAGS: ""
rules:
# extra features when building on `rococo-v1` branch and manual build on PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
- when: always
RUSTFLAGS: "-Cdebug-assertions=y"
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
script:
- time cargo build --release --verbose ${EXTRA_FLAGS}
- time cargo build --release --verbose
- sccache -s
- *pack-artifacts
build-adder-collator:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
script:
- time cargo build --release --verbose -p test-parachain-adder-collator
- sccache -s
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/release/adder-collator ./artifacts/.
- echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
- echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
- echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
- cp -r scripts/docker/* ./artifacts
#### stage: build
check-transaction-versions:
......@@ -213,12 +232,13 @@ generate-impl-guide:
script:
- mdbook build roadmap/implementers-guide
.build-push-docker-image: &build-push-docker-image
.build-push-image: &build-push-image
<<: *kubernetes-env
image: quay.io/buildah/stable
variables:
GIT_STRATEGY: none
# DOCKERFILE: scripts/docker/Dockerfile
# scripts/docker/Dockerfile
DOCKERFILE: Dockerfile
IMAGE_NAME: docker.io/parity/polkadot
DOCKER_USER: ${Docker_Hub_User_Parity}
DOCKER_PASS: ${Docker_Hub_Pass_Parity}
......@@ -236,54 +256,79 @@ generate-impl-guide:
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--build-arg IMAGE_NAME="${IMAGE_NAME}"
--tag "$IMAGE_NAME:$VERSION"
--tag "$IMAGE_NAME:$EXTRATAG" .
--tag "$IMAGE_NAME:$EXTRATAG"
--file ${DOCKERFILE} .
# The job will success only on the protected branch
- echo "$DOCKER_PASS" |
buildah login --username "$DOCKER_USER" --password-stdin docker.io
- buildah info
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
- buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
# pass artifacts to the trigget-simnet job
# this twist is to match the similar logic in substrate
- echo "IMAGE_NAME=${IMAGE_NAME}" > build.env
- echo "VERSION=${EXTRATAG}" >> build.env
after_script:
- buildah logout "$IMAGE_NAME"
# pass artifacts to the trigger-simnet job
- echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/build.env
- echo "IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/build.env
artifacts:
reports:
# this artifact is used in trigger-simnet job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
dotenv: artifacts/build.env
dotenv: ./artifacts/build.env
publish-dockerhub:
publish-polkadot-image:
stage: build
<<: *build-push-docker-image
<<: *build-push-image
rules:
# Don't run on releases - this is handled by the Github Action here:
# .github/workflows/publish-docker-release.yml
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
IMAGE_NAME: docker.io/paritypr/synth-wave
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
IMAGE_NAME: docker.io/parity/rococo
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
# image to be tested with Simnet
IMAGE_NAME: docker.io/paritypr/synth-wave
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
needs:
- job: build-linux-release
artifacts: true
publish-adder-collator-image:
# service image for Simnet
stage: build
<<: *build-push-image
variables:
# scripts/docker/collator.Dockerfile
DOCKERFILE: collator.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
# FIXME: remove me after merging
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
needs:
- job: build-linux-release
- job: build-adder-collator
artifacts: true
after_script:
- buildah logout "$IMAGE_NAME"
# pass artifacts to the trigger-simnet job
- echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" > ./artifacts/collator.env
artifacts:
reports:
# this artifact is used in trigger-simnet job
dotenv: ./artifacts/collator.env
#### stage: publish
publish-s3-release:
publish-s3-release: &publish-s3
stage: publish
needs:
- job: build-linux-release
......@@ -295,10 +340,6 @@ publish-s3-release:
BUCKET: "releases.parity.io"
PREFIX: "polkadot/${ARCH}-${DOCKER_OS}"
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
PREFIX: "rococo/${ARCH}-${DOCKER_OS}"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
......@@ -327,6 +368,14 @@ publish-s3-release:
- aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/
--recursive --human-readable --summarize
publish-s3-adder-collator:
<<: *publish-s3
variables:
PREFIX: "rococo/${ARCH}-${DOCKER_OS}"
needs:
- job: build-adder-collator
artifacts: true
#### stage: deploy
deploy-polkasync-kusama:
......@@ -335,29 +384,36 @@ deploy-polkasync-kusama:
# former .rules-build
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
variables:
POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}"
POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_REF}"
POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}"
allow_failure: true
trigger: "parity/infrastructure/parity-testnet"
trigger-simnet:
stage: deploy
<<: *rules-test
image: paritytech/tools:latest
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: publish-dockerhub
# `build.env` is taken by the triggered job from `publish-dockerhub` job with an
# exact `$VERSION` (`$EXTRATAG` here, i.e. `2643-0.8.29-5f689e0a-6b24dc54`)
- job: publish-polkadot-image
- job: publish-adder-collator-image
# `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here,
# i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
# `collator.env` bears adder-collator unique build tag. In non-triggered builds it
# can be called by `master` tag.
# Simnet uses an image published on PRs with this exact version for triggered runs
# on commits. And parity/rococo:rococo-v1 for runs not launched by this job.
trigger:
project: parity/simnet
branch: master
strategy: depend
# on commits. And parity/rococo:rococo-v1 for the runs not launched by this job.
variables:
TRGR_PROJECT: ${CI_PROJECT_NAME}
TRGR_REF: ${CI_COMMIT_REF_NAME}
# simnet project ID
DWNSTRM_ID: 332
script:
# API trigger for a simnet job
- ./scripts/gitlab/trigger_pipeline.sh
......@@ -12,6 +12,7 @@
- [Runtime Architecture](runtime/README.md)
- [Initializer Module](runtime/initializer.md)
- [Configuration Module](runtime/configuration.md)
- [Shared](runtime/shared.md)
- [Disputes Module](runtime/disputes.md)
- [Paras Module](runtime/paras.md)
- [Scheduler Module](runtime/scheduler.md)
......
......@@ -30,7 +30,7 @@ The other parachains modules are initialized in this order:
1. UMP
1. HRMP
The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared][shared.md] module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module.
The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared](shared.md) module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module.
Set `HasInitialized` to true.
......
......@@ -3,10 +3,11 @@ FROM debian:buster-slim
# metadata
ARG VCS_REF
ARG BUILD_DATE
ARG IMAGE_NAME
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="parity/polkadot" \
io.parity.image.title="${IMAGE_NAME}" \
io.parity.image.description="polkadot: a platform for web3" \
io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \
io.parity.image.revision="${VCS_REF}" \
......@@ -18,7 +19,6 @@ ENV RUST_BACKTRACE 1
# install tools and dependencies
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl1.1 \
ca-certificates \
......@@ -45,4 +45,3 @@ EXPOSE 30333 9933 9944
VOLUME ["/polkadot"]
ENTRYPOINT ["/usr/local/bin/polkadot"]
FROM debian:buster-slim
# metadata
ARG VCS_REF
ARG BUILD_DATE
ARG IMAGE_NAME
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="${IMAGE_NAME}" \
io.parity.image.description="polkadot: a platform for web3" \
io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \
io.parity.image.revision="${VCS_REF}" \
io.parity.image.created="${BUILD_DATE}" \
io.parity.image.documentation="https://github.com/paritytech/polkadot/"
# show backtraces
ENV RUST_BACKTRACE 1
# install tools and dependencies
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl1.1 \
ca-certificates \
curl && \
# apt cleanup
apt-get autoremove -y && \
apt-get clean && \
find /var/lib/apt/lists/ -type f -not -name lock -delete; \
# add user and link ~/.local/share/adder-collator to /data
useradd -m -u 1000 -U -s /bin/sh -d /adder-collator adder-collator && \
mkdir -p /data /adder-collator/.local/share && \
chown -R adder-collator:adder-collator /data && \
ln -s /data /adder-collator/.local/share/polkadot
# add adder-collator binary to docker image
COPY ./adder-collator /usr/local/bin
USER adder-collator
# check if executable works in this container
RUN /usr/local/bin/adder-collator --version
EXPOSE 30333 9933 9944
VOLUME ["/adder-collator"]
ENTRYPOINT ["/usr/local/bin/adder-collator"]
#!/bin/bash
set -eu
# API trigger another project's pipeline
curl --silent \
-X POST \
-F "token=${CI_JOB_TOKEN}" \
-F "ref=master" \
-F "variables[TRGR_PROJECT]=${TRGR_PROJECT}" \
-F "variables[TRGR_REF]=${TRGR_REF}" \
-F "variables[IMAGE_NAME]=${IMAGE_NAME}" \
-F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
-F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
"https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
tee pipeline
PIPELINE_ID=$(cat pipeline | jq ".id")
echo "\nWaiting on ${PIPELINE_ID} status..."
# This part polls for the triggered pipeline status, the native
# `trigger` job does not return this status via API.
# This is a workaround for a Gitlab bug, waits here until
# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
# The timeout is 360 curls with 8 sec interval, roughly an hour.
function get_status() {
curl --silent \
--header "PRIVATE-TOKEN: ${PIPELINE_TOKEN}" \
"https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/pipelines/${PIPELINE_ID}" | \
jq --raw-output ".status";
}
for i in $(seq 1 360); do
STATUS=$(get_status);
echo "Triggered pipeline status is ${STATUS}";
if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
echo "Busy...";
elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
exit 1;
elif [[ ${STATUS} =~ ^(success)$ ]]; then
exit 0;
else
exit 1;
fi
sleep 8;
done
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment