Newer
Older
gabriel klawitter
committed
# .gitlab-ci.yml
#
gabriel klawitter
committed
# pipelines can be triggered manually in the web
# setting DEPLOY_TAG will only deploy the tagged image
#
# please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
gabriel klawitter
committed
gabriel klawitter
committed
- publish
- deploy
workflow:
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
ZOMBIENET_IMAGE: "docker.io/paritypr/zombienet"
VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io"
VAULT_AUTH_PATH: "gitlab-parity-io-jwt"
VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}"
gabriel klawitter
committed
.collect-artifacts: &collect-artifacts
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: on_success
.kubernetes-env: &kubernetes-env
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
interruptible: true
tags:
- kubernetes-parity-build
.docker-env: &docker-env
when:
- runner_system_failure
- unknown_failure
- api_failure
tags:
- linux-docker
.compiler-info: &compiler-info
before_script:
- rustup show
- cargo --version
- sccache -s
.rules-test: &rules-test
# these jobs run always*
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
#### Vault secrets
.vault-secrets: &vault-secrets
secrets:
AWS_ACCESS_KEY_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv
file: false
AWS_SECRET_ACCESS_KEY:
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv
file: false
DOCKER_HUB_USER:
vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv
file: false
DOCKER_HUB_PASS:
vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv
file: false
GITHUB_PR_TOKEN:
vault: cicd/gitlab/parity/GITHUB_PR_TOKEN@kv
file: false
GITHUB_USER:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv
file: false
GITHUB_RELEASE_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv
file: false
GITHUB_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv
MATRIX_ACCESS_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv
file: false
MATRIX_ROOM_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv
PARITYPR_USER:
vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_USER@kv
file: false
PARITYPR_PASS:
vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_PASS@kv
file: false
PIPELINE_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv
file: false
REL_MAN_ROOM_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/REL_MAN_ROOM_ID@kv
file: false
SSH_PRIVATE_KEY:
vault: cicd/gitlab/$CI_PROJECT_PATH/SSH_PRIVATE_KEY@kv
file: false
#### stage: test
check-runtime:
stage: test
variables:
GITLAB_API: "https://gitlab.parity.io/api/v4"
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
script:
- ./scripts/gitlab/check_runtime.sh
allow_failure: true
- cargo +nightly fmt --all -- --check
allow_failure: true
test-deterministic-wasm:
stage: test
<<: *docker-env
script:
- ./scripts/gitlab/test_deterministic_wasm.sh
<<: *docker-env
variables:
RUST_TOOLCHAIN: stable
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
gabriel klawitter
committed
script:
- ./scripts/gitlab/test_linux_stable.sh
# we're using the bin built here, instead of having a parallel `build-linux-release`
# disputes feature is needed for zombie-net parachains malus test
- time cargo build --release --verbose --bin polkadot --features "disputes"
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
- EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
script:
# Check that the node will compile with `runtime-benchmarks` feature flag.
- ./scripts/gitlab/check_runtime_benchmarks.sh
check-no-default-features:
stage: test
<<: *rules-test
<<: *docker-env
<<: *compiler-info
<<: *vault-secrets
script:
# Check that polkadot-cli will compile no default features.
- ./scripts/gitlab/check_no_default_features.sh
- sccache -s
spellcheck:
stage: test
<<: *docker-env
script:
- cargo spellcheck --version
# compare with the commit parent to the PR, given it's from a default branch
- git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
- cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
- time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
$(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
build-adder-collator:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
script:
- time cargo build --release --verbose -p test-parachain-adder-collator
- sccache -s
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/release/adder-collator ./artifacts/.
- echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
- echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
- echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
build-malus:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
script:
- time cargo build --release --verbose -p polkadot-test-malus --features disputes
- sccache -s
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/release/malus ./artifacts/.
- echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
- echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
- echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
- cp -r scripts/* ./artifacts
.check-dependent-project: &check-dependent-project
stage: build
<<: *docker-env
<<: *vault-secrets
script:
- git clone
--depth=1
"--branch=$PIPELINE_SCRIPTS_TAG"
https://github.com/paritytech/pipeline-scripts
- ./pipeline-scripts/check_dependent_project.sh
paritytech
polkadot
--polkadot
"$DEPENDENT_REPO"
"$GITHUB_PR_TOKEN"
check-dependent-cumulus:
<<: *check-dependent-project
variables:
DEPENDENT_REPO: cumulus
check-transaction-versions:
stage: build
<<: *rules-test
<<: *docker-env
<<: *vault-secrets
needs:
- job: test-build-linux-stable
artifacts: true
before_script:
- apt-get -y update; apt-get -y install jq lsof
- npm install --ignore-scripts -g @polkadot/metadata-cmp
# Set git config
- git config remote.origin.url "https://github.com/paritytech/polkadot.git"
- git fetch origin release
script:
- scripts/gitlab/check_extrinsics_ordering.sh
stage: build
<<: *rules-test
<<: *docker-env
name: michaelfbryan/mdbook-docker-image:v0.4.4
- mdbook build roadmap/implementers-guide
build-rustdoc:
stage: build
<<: *docker-env
<<: *rules-test
variables:
SKIP_WASM_BUILD: 1
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
when: on_success
expire_in: 7 days
paths:
- ./crate-docs/
script:
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
- rm -f ./target/doc/.lock
- mv ./target/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot
- chown -R nonroot:nonroot ./crate-docs
- echo "<meta http-equiv=refresh content=0;url=polkadot_service/index.html>" > ./crate-docs/index.html
- sccache -s
<<: *kubernetes-env
image: quay.io/buildah/stable
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
before_script: &check-versions
- test -s ./artifacts/VERSION || exit 1
- test -s ./artifacts/EXTRATAG || exit 1
- VERSION="$(cat ./artifacts/VERSION)"
- EXTRATAG="$(cat ./artifacts/EXTRATAG)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- test "$DOCKER_USER" -a "$DOCKER_PASS" ||
( echo "no docker credentials provided"; exit 1 )
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--tag "$IMAGE_NAME:$EXTRATAG"
--file ${DOCKERFILE} .
# The job will success only on the protected branch
- echo "$DOCKER_PASS" |
buildah login --username "$DOCKER_USER" --password-stdin docker.io
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
- buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
gabriel klawitter
committed
after_script:
variables:
<<: *image-variables
# scripts/dockerfiles/polkadot_injected_debug.Dockerfile
DOCKERFILE: dockerfiles/polkadot_injected_debug.Dockerfile
IMAGE_NAME: docker.io/paritypr/synth-wave
# Don't run on releases - this is handled by the Github Action here:
# .github/workflows/publish-docker-release.yml
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
IMAGE_NAME: docker.io/parity/rococo
DOCKER_USER: ${DOCKER_HUB_USER}
DOCKER_PASS: ${DOCKER_HUB_PASS}
# pass artifacts to the zombienet-tests job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
- echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
- echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
artifacts:
reports:
# this artifact is used in zombienet-tests job
publish-adder-collator-image:
# service image for Simnet
stage: build
<<: *build-push-image
variables:
<<: *image-variables
# scripts/dockerfiles/collator_injected.Dockerfile
DOCKERFILE: dockerfiles/collator_injected.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander
needs:
- job: build-adder-collator
artifacts: true
after_script:
# pass artifacts to the zombienet-tests job
- echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
- echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
# this artifact is used in zombienet-tests job
dotenv: ./artifacts/collator.env
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
publish-malus-image:
# service image for Simnet
stage: build
<<: *build-push-image
variables:
<<: *image-variables
# scripts/dockerfiles/malus.Dockerfile
DOCKERFILE: dockerfiles/malus.Dockerfile
IMAGE_NAME: docker.io/paritypr/malus
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
needs:
- job: build-malus
artifacts: true
after_script:
- buildah logout "$IMAGE_NAME"
# pass artifacts to the zombienet-tests job
- echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env
- echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env
artifacts:
reports:
# this artifact is used in zombienet-tests job
dotenv: ./artifacts/malus.env
stage: build
when: manual
variables:
RUNTIME: polkadot
artifacts:
paths:
- ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
script:
- ./scripts/run_benches_for_runtime.sh $RUNTIME
- git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
# uses the "shell" executors
tags:
- weights
update_kusama_weights:
<<: *update-weights
variables:
RUNTIME: kusama
update_westend_weights:
<<: *update-weights
variables:
RUNTIME: westend
artifacts: true
<<: *kubernetes-env
variables:
GIT_STRATEGY: none
- if: $CI_PIPELINE_SOURCE == "schedule"
before_script:
- *check-versions
- echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
- aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
- echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
- find ./artifacts -type f | while read file; do
name="${file#./artifacts/}";
aws s3api copy-object
--copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
--bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
- |
cat <<-EOM
|
| - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
| - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
- aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
--recursive --human-readable --summarize
gabriel klawitter
committed
publish-rustdoc:
stage: publish
<<: *kubernetes-env
<<: *vault-secrets
image: paritytech/tools:latest
variables:
GIT_DEPTH: 100
rules:
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"
# `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
# artifacts from being dowloaded by this job.
needs:
- job: build-rustdoc
artifacts: true
script:
- rm -rf /tmp/*
# Set git config
- rm -rf .git/config
- git config user.email "devops-team@parity.io"
- git config user.name "${GITHUB_USER}"
- git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/polkadot.git"
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
- git fetch origin gh-pages
# Save README and docs
- cp -r ./crate-docs/ /tmp/doc/
- cp README.md /tmp/doc/
- git checkout gh-pages
# Remove everything and restore generated docs and README
- rm -rf ./*
- mv /tmp/doc/* .
# Upload files
- git add --all --force
# `git commit` has an exit code of > 0 if there is nothing to commit.
# This causes GitLab to exit immediately and marks this job failed.
# We don't want to mark the entire job failed if there's nothing to
# publish though, hence the `|| true`.
- git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
echo "___Nothing to commit___"
- git push origin gh-pages --force
- echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
deploy-polkasync-kusama:
stage: deploy
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
variables:
gabriel klawitter
committed
POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}"
POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}"
allow_failure: true
gabriel klawitter
committed
trigger: "parity/infrastructure/parity-testnet"
zombienet-tests-parachains-smoke-test:
image: "${ZOMBIENET_IMAGE}"
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: publish-polkadot-image
- job: publish-malus-image
- job: publish-adder-collator-image
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
GH_DIR: 'https://github.com/paritytech/polkadot/tree/bernhard-malus-fx-zombienet/zombienet_tests/parachains'
# FIXME: use the master after the merge of the malus pr
# GH_DIR: 'https://github.com/paritytech/polkadot/tree/master/zombienet_tests/parachains'
before_script:
- echo "Zombie-net Tests Config"
- echo "${ZOMBIENET_IMAGE_NAME}"
- echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
- echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
- echo "${GH_DIR}"
- export DEBUG=zombie,zombie::network-node
- export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
- export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
- export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
script:
- /home/nonroot/zombie-net/scripts/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}"
--test="0001-parachains-smoke-test.feature"
allow_failure: true
retry: 2
tags:
- zombienet-polkadot-integration-test
zombienet-tests-malus-dispute-valid:
stage: deploy
image: "${ZOMBIENET_IMAGE}"
<<: *kubernetes-env
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: publish-polkadot-image
- job: publish-malus-image
- job: publish-adder-collator-image
variables:
GH_DIR: 'https://github.com/paritytech/polkadot/tree/bernhard-malus-fx-zombienet/node/malus/integrationtests'
# FIXME: use the master after the merge of the malus pr
# GH_DIR: 'https://github.com/paritytech/polkadot/tree/master/node/malus/integrationtests'
- echo "Zombie-net Tests Config"
- echo "${ZOMBIENET_IMAGE_NAME}"
- echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
- echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
- export DEBUG=zombie,zombie::network-node
- export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
- export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
- export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
- /home/nonroot/zombie-net/scripts/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}"
--test="0001-dispute-valid-block.feature"
- zombienet-polkadot-integration-test