# .gitlab-ci.yml # # substrate # # pipelines can be triggered manually in the web # SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a # simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ # # my-example-job: # stage: test # One of the stages listed below this job (required) # image: paritytech/tools:latest # Any docker image (required) # allow_failure: true # Allow the pipeline to continue if this job fails (default: false) # needs: # - job: test-linux # Any jobs that are required to run before this job (optional) # variables: # MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) # script: # - echo "List of shell commands to run in your job" # - echo "You can also just specify a script here, like so:" # - ./.maintain/gitlab/my_amazing_script.sh stages: - check - test - build - publish - deploy - flaming-fir workflow: rules: - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH variables: &default-vars GIT_STRATEGY: fetch GIT_DEPTH: 100 CARGO_INCREMENTAL: 0 DOCKER_OS: "debian:stretch" ARCH: "x86_64" # FIXME set to release CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.11" CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" default: cache: {} .collect-artifacts: &collect-artifacts artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success expire_in: 7 days paths: - artifacts/ .kubernetes-build: &kubernetes-build tags: - kubernetes-parity-build interruptible: true .docker-env: &docker-env image: paritytech/ci-linux:production before_script: - rustup show - cargo --version - sccache -s retry: max: 2 when: - runner_system_failure - unknown_failure - api_failure interruptible: true tags: - linux-docker .test-refs: &test-refs rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .build-refs: &build-refs rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 #### stage: .pre skip-if-draft: image: paritytech/tools:latest <<: *kubernetes-build stage: .pre rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - echo "Commit message is ${CI_COMMIT_MESSAGE}" - echo "Ref is ${CI_COMMIT_REF_NAME}" - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - ./.maintain/gitlab/skip_if_draft.sh #### stage: check check-runtime: stage: check image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs variables: <<: *default-vars GITLAB_API: "https://gitlab.parity.io/api/v4" GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" script: - ./.maintain/gitlab/check_runtime.sh allow_failure: true check-signed-tag: stage: check image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - ./.maintain/gitlab/check_signed.sh check-line-width: stage: check image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - ./.maintain/gitlab/check_line_width.sh allow_failure: true test-dependency-rules: stage: check image: paritytech/tools:latest <<: *kubernetes-build script: - .maintain/ensure-deps.sh test-prometheus-alerting-rules: stage: check image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_BRANCH changes: - .gitlab-ci.yml - .maintain/monitoring/**/* script: - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml #### stage: test cargo-audit: stage: test <<: *docker-env rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - cargo audit allow_failure: true cargo-deny: stage: test <<: *docker-env rules: - if: $CI_COMMIT_MESSAGE =~ /skip-checks/ when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml after_script: - echo "___The complete log is in the artifacts___" - cargo deny check -c .maintain/deny.toml 2> deny.log artifacts: name: $CI_COMMIT_SHORT_SHA expire_in: 3 days when: always paths: - deny.log cargo-check-benches: stage: test <<: *docker-env <<: *test-refs script: - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small - cargo run --release -p node-bench -- ::trie::read::small - sccache -s cargo-check-subkey: stage: test <<: *docker-env <<: *test-refs script: - cd ./bin/utils/subkey - SKIP_WASM_BUILD=1 time cargo check --release - sccache -s test-deterministic-wasm: stage: test <<: *docker-env <<: *test-refs variables: <<: *default-vars WASM_BUILD_NO_COLOR: 1 script: # build runtime - cargo build --verbose --release -p node-runtime # make checksum - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 # clean up – FIXME: can we reuse some of the artifacts? - cargo clean # build again - cargo build --verbose --release -p node-runtime # confirm checksum - sha256sum -c checksum.sha256 - sccache -s test-linux-stable: &test-linux stage: test <<: *docker-env <<: *test-refs variables: <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" RUST_BACKTRACE: 1 WASM_BUILD_NO_COLOR: 1 script: # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - sccache -s unleash-check: stage: test <<: *docker-env rules: - if: $CI_COMMIT_MESSAGE =~ /skip-checks/ when: never # .test-refs - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} test-frame-examples-compile-to-wasm: # into one job stage: test <<: *docker-env <<: *test-refs variables: <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y RUST_BACKTRACE: 1 script: - cd frame/example-offchain-worker/ - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - cd ../example - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - sccache -s test-linux-stable-int: <<: *test-linux script: - echo "___Logs will be partly shown at the end in case of failure.___" - echo "___Full log will be saved to the job artifacts only in case of failure.___" - WASM_BUILD_NO_COLOR=1 RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace time cargo test -p node-cli --release --verbose --locked -- --ignored &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - sccache -s after_script: - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log artifacts: name: $CI_COMMIT_SHORT_SHA when: on_failure expire_in: 3 days paths: - ${CI_COMMIT_SHORT_SHA}_int_failure.log check-web-wasm: stage: test <<: *docker-env <<: *test-refs script: # WASM support is in progress. As more and more crates support WASM, we # should add entries here. See https://github.com/paritytech/substrate/issues/2416 # Note: we don't need to test crates imported in `bin/node/cli` - time cargo build --manifest-path=client/consensus/aura/Cargo.toml --target=wasm32-unknown-unknown --features getrandom # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way. - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - sccache -s test-full-crypto-feature: stage: test <<: *docker-env <<: *test-refs variables: <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y RUST_BACKTRACE: 1 script: - cd primitives/core/ - time cargo +nightly build --verbose --no-default-features --features full_crypto - cd ../application-crypto - time cargo +nightly build --verbose --no-default-features --features full_crypto - sccache -s cargo-check-macos: stage: test # shell runner on mac ignores the image set in *docker-env <<: *docker-env <<: *test-refs script: - SKIP_WASM_BUILD=1 time cargo check --release - sccache -s tags: - osx #### stage: build check-polkadot-companion-status: stage: build image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - ./.maintain/gitlab/check_polkadot_companion_status.sh check-polkadot-companion-build: stage: build <<: *docker-env <<: *test-refs needs: - job: test-linux-stable-int artifacts: false script: - ./.maintain/gitlab/check_polkadot_companion_build.sh after_script: - cd polkadot && git rev-parse --abbrev-ref HEAD allow_failure: true test-browser-node: stage: build <<: *docker-env <<: *test-refs needs: - job: check-web-wasm artifacts: false variables: <<: *default-vars CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1" CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner" WASM_BINDGEN_TEST_TIMEOUT: 120 script: - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget build-linux-substrate: &build-binary stage: build <<: *collect-artifacts <<: *docker-env rules: # .build-refs with manual on PRs - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: manual allow_failure: true needs: - job: test-linux-stable artifacts: false before_script: - mkdir -p ./artifacts/substrate/ script: - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - mv ./target/release/substrate ./artifacts/substrate/. - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; else ./artifacts/substrate/substrate --version | sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | tee ./artifacts/substrate/VERSION; fi - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - printf '\n# building node-template\n\n' - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - sccache -s build-linux-subkey: &build-subkey stage: build <<: *collect-artifacts <<: *docker-env rules: # .build-refs with manual on PRs - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: manual allow_failure: true needs: - job: cargo-check-subkey artifacts: false before_script: - mkdir -p ./artifacts/subkey script: - cd ./bin/utils/subkey - SKIP_WASM_BUILD=1 time cargo build --release --verbose - cd - - mv ./target/release/subkey ./artifacts/subkey/. - echo -n "Subkey version = " - ./artifacts/subkey/subkey --version | sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | tee ./artifacts/subkey/VERSION; - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ - sccache -s build-macos-subkey: <<: *build-subkey tags: - osx build-rust-doc: stage: build <<: *docker-env <<: *test-refs needs: - job: test-linux-stable artifacts: false variables: <<: *default-vars RUSTFLAGS: -Dwarnings artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" when: on_success expire_in: 7 days paths: - ./crate-docs/ script: - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds - SKIP_WASM_BUILD=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" time cargo +nightly doc --no-deps --workspace --all-features --verbose - mv ./target/doc ./crate-docs - echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html - sccache -s #### stage: publish .build-push-docker-image: &build-push-docker-image <<: *build-refs <<: *kubernetes-build image: quay.io/buildah/stable variables: &docker-build-vars <<: *default-vars GIT_STRATEGY: none DOCKERFILE: $PRODUCT.Dockerfile IMAGE_NAME: docker.io/parity/$PRODUCT before_script: - cd ./artifacts/$PRODUCT/ - VERSION="$(cat ./VERSION)" - echo "${PRODUCT} version = ${VERSION}" - test -z "${VERSION}" && exit 1 script: - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" || ( echo "no docker credentials provided"; exit 1 ) - buildah bud --format=docker --build-arg VCS_REF="${CI_COMMIT_SHA}" --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" --tag "$IMAGE_NAME:$VERSION" --tag "$IMAGE_NAME:latest" --file "$DOCKERFILE" . - echo "$Docker_Hub_Pass_Parity" | buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io - buildah info - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - buildah push --format=v2s2 "$IMAGE_NAME:latest" - buildah logout "$IMAGE_NAME" publish-docker-substrate: stage: publish <<: *build-push-docker-image # collect VERSION artifact here to pass it on to kubernetes <<: *collect-artifacts needs: - job: build-linux-substrate artifacts: true variables: <<: *docker-build-vars PRODUCT: substrate after_script: # only VERSION information is needed for the deployment - find ./artifacts/ -depth -not -name VERSION -type f -delete publish-docker-subkey: stage: publish <<: *build-push-docker-image needs: - job: build-linux-subkey artifacts: true variables: <<: *docker-build-vars PRODUCT: subkey publish-s3-release: stage: publish <<: *build-refs <<: *kubernetes-build needs: - job: build-linux-substrate artifacts: true - job: build-linux-subkey artifacts: true image: paritytech/awscli:latest variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" PREFIX: "substrate/${ARCH}-${DOCKER_OS}" script: - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - echo "update objects in latest path" - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ after_script: - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ --recursive --human-readable --summarize publish-s3-doc: stage: publish image: paritytech/awscli:latest allow_failure: true needs: - job: build-rust-doc artifacts: true - job: build-linux-substrate artifacts: false <<: *build-refs <<: *kubernetes-build variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" PREFIX: "substrate-rustdoc" script: - test -r ./crate-docs/index.html || ( echo "./crate-docs/index.html not present, build:rust:doc:release job not complete"; exit 1 ) - aws s3 sync --delete --size-only --only-show-errors ./crate-docs/ s3://${BUCKET}/${PREFIX}/ after_script: - aws s3 ls s3://${BUCKET}/${PREFIX}/ --human-readable --summarize publish-draft-release: stage: publish image: paritytech/tools:latest rules: - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - ./.maintain/gitlab/publish_draft_release.sh allow_failure: true publish-to-crates-io: stage: publish <<: *docker-env rules: - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF} allow_failure: true #### stage: deploy deploy-prometheus-alerting-rules: stage: deploy needs: - job: test-prometheus-alerting-rules artifacts: false interruptible: true retry: 1 tags: - kubernetes-parity-build image: paritytech/kubetools:latest environment: name: parity-mgmt-polkadot-alerting variables: NAMESPACE: monitoring PROMETHEUSRULE: prometheus-k8s-rules-polkadot-alerting RULES: .maintain/monitoring/alerting-rules/alerting-rules.yaml script: - echo "deploying prometheus alerting rules" - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE} --type=merge --patch "$(sed 's/^/ /;1s/^/spec:\n/' ${RULES})" rules: - if: $CI_COMMIT_REF_NAME == "master" changes: - .gitlab-ci.yml - .maintain/monitoring/**/* trigger-simnet: stage: deploy rules: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" needs: - job: publish-docker-substrate artifacts: false trigger: project: parity/simnet branch: master strategy: depend #### stage: .post check-labels: stage: .post image: paritytech/tools:latest <<: *kubernetes-build rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - ./.maintain/gitlab/check_labels.sh