Newer
Older
# .gitlab-ci.yml
#
# substrate
#
# pipelines can be triggered manually in the web
# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
#
# my-example-job:
# stage: test # One of the stages listed below this job (required)
# image: paritytech/tools:latest # Any docker image (required)
# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
# needs:
# - job: test-linux # Any jobs that are required to run before this job (optional)
# variables:
# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
# script:
# - echo "List of shell commands to run in your job"
# - echo "You can also just specify a script here, like so:"
# - ./.maintain/gitlab/my_amazing_script.sh
- publish
workflow:
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
CI_IMAGE: "paritytech/ci-linux:production"
CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.11"
CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder"
.collect-artifacts: &collect-artifacts
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: on_success
expire_in: 7 days
.kubernetes-build: &kubernetes-build
tags:
- kubernetes-parity-build

Vladimir Istyufeev
committed
.rust-info-script: &rust-info-script
- rustup show
- cargo --version
- sccache -s

Vladimir Istyufeev
committed
- *rust-info-script
- runner_system_failure
- unknown_failure
- api_failure
.test-refs: &test-refs
rules:
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
.test-refs-no-trigger: &test-refs-no-trigger
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1

Vladimir Istyufeev
committed
.test-refs-no-trigger-prs-only: &test-refs-no-trigger-prs-only
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
.build-refs: &build-refs
rules:
# .publish-refs with manual on PRs
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: manual
allow_failure: true
.publish-refs: &publish-refs
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
.nightly-pipeline: &nightly-pipeline
rules:
# this job runs only on nightly pipeline with the mentioned variable, against `master` branch
- if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"

Vladimir Istyufeev
committed
.merge-ref-into-master-script: &merge-ref-into-master-script
- git fetch origin +master:master
- git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME
- git checkout master
- git config user.email "ci@gitlab.parity.io"
- git merge $CI_COMMIT_REF_NAME --verbose --no-edit
.cargo-check-benches-script: &cargo-check-benches-script
- mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
- SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
- 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
- 'cargo run --release -p node-bench -- ::trie::read::small --json
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
- sccache -s
gabriel klawitter
committed
stage: .pre
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- echo "Commit message is ${CI_COMMIT_MESSAGE}"
- echo "Ref is ${CI_COMMIT_REF_NAME}"
- echo "pipeline source is ${CI_PIPELINE_SOURCE}"
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
gabriel klawitter
committed
variables:
gabriel klawitter
committed
GITLAB_API: "https://gitlab.parity.io/api/v4"
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
- ./.maintain/gitlab/check_runtime.sh
check-signed-tag:
<<: *kubernetes-build
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- ./.maintain/gitlab/check_signed.sh
check-line-width:
<<: *kubernetes-build
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- ./.maintain/gitlab/check_line_width.sh
test-dependency-rules:
stage: check
image: paritytech/tools:latest
<<: *kubernetes-build
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
test-prometheus-alerting-rules:
stage: check
image: paritytech/tools:latest
<<: *kubernetes-build
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_BRANCH
changes:
- .gitlab-ci.yml
- .maintain/monitoring/**/*
script:
- promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
- cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
script:
- cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
after_script:
- echo "___The complete log is in the artifacts___"
- cargo deny check -c .maintain/deny.toml 2> deny.log
artifacts:
name: $CI_COMMIT_SHORT_SHA
expire_in: 3 days
when: always
paths:
- deny.log
# FIXME: Temorarily allow to fail.
allow_failure: true
cargo-check-benches:
stage: test
<<: *docker-env

Vladimir Istyufeev
committed
<<: *collect-artifacts

Vladimir Istyufeev
committed
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
- *cargo-check-benches-script
cargo-check-benches-merged:
stage: test
<<: *docker-env
<<: *test-refs-no-trigger-prs-only
<<: *collect-artifacts
before_script:
- *merge-ref-into-master-script
- *rust-info-script
script:
- *cargo-check-benches-script
node-bench-regression-guard:
# it's not belong to `build` semantically, but dag jobs can't depend on each other
# within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
# more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
stage: build
<<: *docker-env
<<: *test-refs-no-trigger-prs-only
needs:
- job: cargo-check-benches-merged
artifacts: true
- project: $CI_PROJECT_PATH
job: cargo-check-benches
ref: master
artifacts: true
variables:
CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
before_script: [""]
script:
- 'node-bench-regression-guard --reference artifacts/benches/master-*
--compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
cargo-check-subkey:
stage: test
<<: *docker-env
- SKIP_WASM_BUILD=1 time cargo check --release
cargo-check-try-runtime:
stage: test
<<: *docker-env
<<: *test-refs
script:
- time cargo check --features try-runtime
- sccache -s
Benjamin Kampmann
committed
test-deterministic-wasm:
stage: test
<<: *docker-env
Benjamin Kampmann
committed
variables:
<<: *default-vars
Benjamin Kampmann
committed
script:
# build runtime
- cargo build --verbose --release -p node-runtime
Benjamin Kampmann
committed
# make checksum
- sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
Benjamin Kampmann
committed
# clean up – FIXME: can we reuse some of the artifacts?
- cargo clean
# build again
- cargo build --verbose --release -p node-runtime
Benjamin Kampmann
committed
# confirm checksum
- sha256sum -c checksum.sha256
- sccache -s
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
RUST_BACKTRACE: 1
WASM_BUILD_NO_COLOR: 1
script:
# this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
- time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
- time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml # does not reuse cache 1 min 44 sec
- SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
unleash-check:
stage: test
<<: *docker-env
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
- cargo unleash check ${CARGO_UNLEASH_PKG_DEF}
test-frame-examples-compile-to-wasm:
stage: test
<<: *docker-env
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1
script:
- cd frame/example-offchain-worker/
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
- cd ../example
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
- sccache -s
- echo "___Logs will be partly shown at the end in case of failure.___"
- echo "___Full log will be saved to the job artifacts only in case of failure.___"
- WASM_BUILD_NO_COLOR=1
RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
time cargo test -p node-cli --release --verbose --locked -- --ignored
&> ${CI_COMMIT_SHORT_SHA}_int_failure.log
after_script:
- awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
artifacts:
name: $CI_COMMIT_SHORT_SHA
when: on_failure
expire_in: 3 days
paths:
- ${CI_COMMIT_SHORT_SHA}_int_failure.log
check-web-wasm:
stage: test
script:
# WASM support is in progress. As more and more crates support WASM, we
# should add entries here. See https://github.com/paritytech/substrate/issues/2416
# Note: we don't need to test crates imported in `bin/node/cli`
- time cargo build --manifest-path=client/consensus/aura/Cargo.toml --target=wasm32-unknown-unknown --features getrandom
# Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
- time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown
# with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
stage: test
<<: *docker-env
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1
script:
- cd primitives/core/
- time cargo +nightly build --verbose --no-default-features --features full_crypto
- cd ../application-crypto
- time cargo +nightly build --verbose --no-default-features --features full_crypto
- sccache -s
Denis S. Soldatov aka General-Beck
committed
cargo-check-macos:
stage: test
# shell runner on mac ignores the image set in *docker-env
Denis S. Soldatov aka General-Beck
committed
<<: *docker-env
Denis S. Soldatov aka General-Beck
committed
script:
- SKIP_WASM_BUILD=1 time cargo check --release
Denis S. Soldatov aka General-Beck
committed
- sccache -s
tags:
- osx
check-polkadot-companion-status:
stage: build
image: paritytech/tools:latest
<<: *kubernetes-build
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
script:
- ./.maintain/gitlab/check_polkadot_companion_status.sh
check-polkadot-companion-build:
stage: build
<<: *docker-env
needs:
- job: test-linux-stable-int
artifacts: false
script:
- ./.maintain/gitlab/check_polkadot_companion_build.sh
after_script:
- cd polkadot && git rev-parse --abbrev-ref HEAD
test-browser-node:
stage: build
<<: *docker-env
needs:
- job: check-web-wasm
artifacts: false
variables:
<<: *default-vars
CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1"
CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner"
WASM_BINDGEN_TEST_TIMEOUT: 120
- cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing
before_script:
- mkdir -p ./artifacts/substrate/
script:
- WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
- mv ./target/release/substrate ./artifacts/substrate/.
- echo -n "Substrate version = "
- if [ "${CI_COMMIT_TAG}" ]; then
echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
Denis S. Soldatov aka General-Beck
committed
sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
- sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
- ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
- cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
Denis S. Soldatov aka General-Beck
committed
build-linux-subkey: &build-subkey
stage: build
<<: *collect-artifacts
<<: *docker-env
- cd ./bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo build --release --verbose
Max Inden
committed
- cd -
- mv ./target/release/subkey ./artifacts/subkey/.
Denis S. Soldatov aka General-Beck
committed
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
Denis S. Soldatov aka General-Beck
committed
build-macos-subkey:
<<: *build-subkey
tags:
- osx
build-rust-doc:
stage: build
<<: *docker-env
<<: *test-refs
needs:
- job: test-linux-stable
artifacts: false
variables:
<<: *default-vars
SKIP_WASM_BUILD: 1
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
when: on_success
expire_in: 7 days
paths:
- ./crate-docs/
script:
- RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
time cargo +nightly doc --no-deps --workspace --all-features --verbose
- mv ./target/doc ./crate-docs
- echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html
- sccache -s
allow_failure: true
#### stage: publish
variables: &docker-build-vars
<<: *default-vars
GIT_STRATEGY: none
DOCKERFILE: $PRODUCT.Dockerfile
script:
- test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
( echo "no docker credentials provided"; exit 1 )
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--tag "$IMAGE_NAME:$VERSION"
--tag "$IMAGE_NAME:latest"
--file "$DOCKERFILE" .
buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
- buildah push --format=v2s2 "$IMAGE_NAME:latest"
- buildah logout "$IMAGE_NAME"
# pass artifacts to the trigger-simnet job
- echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/$PRODUCT/build.env
- echo "IMAGE_TAG=${VERSION}" >> ./artifacts/$PRODUCT/build.env
publish-docker-substrate:
needs:
- job: build-linux-substrate
artifacts: true
artifacts:
reports:
# this artifact is used in trigger-simnet job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
publish-docker-subkey:
stage: publish
<<: *build-push-docker-image
needs:
- job: build-linux-subkey
artifacts: true
needs:
- job: build-linux-substrate
artifacts: true
- job: build-linux-subkey
artifacts: true
image: paritytech/awscli:latest
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
script:
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
- echo "update objects in latest path"
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
--recursive --human-readable --summarize
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
publish-s3-doc:
stage: publish
image: paritytech/awscli:latest
allow_failure: true
needs:
- job: build-rust-doc
artifacts: true
- job: build-linux-substrate
artifacts: false
<<: *publish-refs
<<: *kubernetes-build
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "substrate-rustdoc"
script:
- test -r ./crate-docs/index.html || (
echo "./crate-docs/index.html not present, build:rust:doc:release job not complete";
exit 1
)
- aws s3 sync --delete --size-only --only-show-errors
./crate-docs/ s3://${BUCKET}/${PREFIX}/
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/
--human-readable --summarize
publish-draft-release:
stage: publish
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- ./.maintain/gitlab/publish_draft_release.sh
publish-to-crates-io:
stage: publish
<<: *docker-env
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
Benjamin Kampmann
committed
- cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF}
deploy-prometheus-alerting-rules:
stage: deploy
needs:
- job: test-prometheus-alerting-rules
artifacts: false
interruptible: true
retry: 1
tags:
- kubernetes-parity-build
image: paritytech/kubetools:latest
environment:
name: parity-mgmt-polkadot-alerting
variables:
NAMESPACE: monitoring
PROMETHEUSRULE: prometheus-k8s-rules-polkadot-alerting
RULES: .maintain/monitoring/alerting-rules/alerting-rules.yaml
script:
- echo "deploying prometheus alerting rules"
Cecile Tonglet
committed
- kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
--type=merge --patch "$(sed 's/^/ /;1s/^/spec:\n/' ${RULES})"
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_REF_NAME == "master"
changes:
- .gitlab-ci.yml
- .maintain/monitoring/**/*
trigger-simnet:
stage: deploy
image: paritytech/tools:latest
rules:
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"
# `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here,
# i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
variables:
TRGR_PROJECT: ${CI_PROJECT_NAME}
TRGR_REF: ${CI_COMMIT_REF_NAME}
# simnet project ID
DWNSTRM_ID: 332
script:
# API trigger for a simnet job
- ./scripts/gitlab/trigger_pipeline.sh