From 66e1092229c94ce59518c5ac579f978255bd87c9 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Wed, 19 Oct 2022 16:31:42 +0200 Subject: [PATCH] [ci] Split .gitlab-ci.yml (#1701) * [WIP][ci] Split .gitlab-ci.yml * comment unused includes * fix typo * fix needs * add all stages * exclude zombienet from check --- cumulus/.github/pr-custom-review.yml | 1 + cumulus/.gitlab-ci.yml | 583 ++---------------- .../scripts/ci/gitlab/pipeline/benchmarks.yml | 61 ++ cumulus/scripts/ci/gitlab/pipeline/build.yml | 113 ++++ .../ci/gitlab/pipeline/integration_tests.yml | 2 + .../scripts/ci/gitlab/pipeline/publish.yml | 117 ++++ cumulus/scripts/ci/gitlab/pipeline/test.yml | 79 +++ .../scripts/ci/gitlab/pipeline/zombienet.yml | 101 +++ 8 files changed, 517 insertions(+), 540 deletions(-) create mode 100644 cumulus/scripts/ci/gitlab/pipeline/benchmarks.yml create mode 100644 cumulus/scripts/ci/gitlab/pipeline/build.yml create mode 100644 cumulus/scripts/ci/gitlab/pipeline/integration_tests.yml create mode 100644 cumulus/scripts/ci/gitlab/pipeline/publish.yml create mode 100644 cumulus/scripts/ci/gitlab/pipeline/test.yml create mode 100644 cumulus/scripts/ci/gitlab/pipeline/zombienet.yml diff --git a/cumulus/.github/pr-custom-review.yml b/cumulus/.github/pr-custom-review.yml index 18e2482b2a4..99155388877 100644 --- a/cumulus/.github/pr-custom-review.yml +++ b/cumulus/.github/pr-custom-review.yml @@ -29,6 +29,7 @@ rules: check_type: changed_files condition: include: ^\.gitlab-ci\.yml|^scripts/ci/.*|^\.github/.* + exclude: ^scripts/ci/gitlab/pipeline/zombienet.yml$ min_approvals: 2 teams: - ci diff --git a/cumulus/.gitlab-ci.yml b/cumulus/.gitlab-ci.yml index dcb7de25f2e..933c0eb196c 100644 --- a/cumulus/.gitlab-ci.yml +++ b/cumulus/.gitlab-ci.yml @@ -11,7 +11,8 @@ stages: - benchmarks-build - benchmarks-run - publish - - integration-test + - integration-tests + - zombienet default: interruptible: true @@ -31,7 +32,7 @@ variables: ARCH: "x86_64" ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.2.80" -.collect-artifacts: &collect-artifacts +.collect-artifacts: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success @@ -39,27 +40,21 @@ variables: paths: - ./artifacts/ -.rust-info-script: &rust-info-script - - rustup show - - cargo --version - - rustup +nightly show - - cargo +nightly --version - - bash --version - # collecting vars for pipeline stopper # they will be used if the job fails -.pipeline-stopper-vars: &pipeline-stopper-vars +.pipeline-stopper-vars: + before_script: - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "PR_NUM=${CI_COMMIT_REF_NAME}" >> pipeline-stopper.env -.pipeline-stopper-artifacts: &pipeline-stopper-artifacts +.pipeline-stopper-artifacts: artifacts: reports: dotenv: pipeline-stopper.env -.common-refs: &common-refs +.common-refs: # these jobs run always* rules: - if: $CI_PIPELINE_SOURCE == "schedule" @@ -69,29 +64,29 @@ variables: - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1 -.pr-refs: &pr-refs +.pr-refs: # these jobs run always* rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.publish-refs: &publish-refs +.publish-refs: rules: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 # run benchmarks manually only on release-parachains-v* branch -.benchmarks-manual-refs: &benchmarks-manual-refs +.benchmarks-manual-refs: rules: - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 when: manual # run benchmarks only on release-parachains-v* branch -.benchmarks-refs: &benchmarks-refs +.benchmarks-refs: rules: - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 -.zombienet-refs: &zombienet-refs +.zombienet-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -100,540 +95,48 @@ variables: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.docker-env: &docker-env +.docker-env: image: "${CI_IMAGE}" before_script: - - *rust-info-script + - rustup show + - cargo --version + - rustup +nightly show + - cargo +nightly --version + - bash --version tags: - linux-docker -.kubernetes-env: &kubernetes-env +.kubernetes-env: image: "${CI_IMAGE}" tags: - kubernetes-parity-build -#### stage: test - -test-linux-stable: - stage: test - <<: *docker-env - <<: *common-refs - <<: *pipeline-stopper-artifacts - before_script: - - *rust-info-script - - *pipeline-stopper-vars - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo nextest run --all --release --locked --run-ignored all - -test-doc: - stage: test - <<: *docker-env - <<: *common-refs - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --doc - -check-runtime-benchmarks: - stage: test - <<: *docker-env - <<: *common-refs - script: - # Check that the node will compile with `runtime-benchmarks` feature flag. - - time cargo check --all --features runtime-benchmarks - # Check that parachain-template will compile with `runtime-benchmarks` feature flag. - - time cargo check -p parachain-template-node --features runtime-benchmarks - -cargo-check-try-runtime: - stage: test - <<: *docker-env - <<: *common-refs - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-runtime-benchmarks - artifacts: false - script: - # Check that the node will compile with `try-runtime` feature flag. - - time cargo check --all --features try-runtime - # Check that parachain-template will compile with `try-runtime` feature flag. - - time cargo check -p parachain-template-node --features try-runtime - -check-rustdoc: - stage: test - <<: *docker-env - <<: *common-refs - variables: - SKIP_WASM_BUILD: 1 - RUSTDOCFLAGS: "-Dwarnings" - script: - - time cargo +nightly doc --workspace --all-features --verbose --no-deps - -cargo-check-benches: - stage: test - <<: *docker-env - <<: *common-refs - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-rustdoc - artifacts: false - script: - - time cargo check --all --benches - -#### stage: build - -build-linux-stable: - stage: build - <<: *docker-env - <<: *collect-artifacts - <<: *common-refs - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-rustdoc - artifacts: false - script: - - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - - time cargo build --release --locked --bin polkadot-parachain - - echo "___Packing the artifacts___" - - mkdir -p ./artifacts - - mv ./target/release/polkadot-parachain ./artifacts/. - - echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___" - - echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION - -build-test-parachain: - stage: build - <<: *docker-env - <<: *collect-artifacts - <<: *common-refs - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-rustdoc - artifacts: false - script: - - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - - time cargo +nightly build --release --locked --bin test-parachain - - echo "___Packing the artifacts___" - - mkdir -p ./artifacts - - mv ./target/release/test-parachain ./artifacts/. - - mkdir -p ./artifacts/zombienet - - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/. - -# build runtime only if files in $RUNTIME_PATH/$RUNTIME_NAME were changed -.build-runtime-template: &build-runtime-template - stage: build - <<: *docker-env - <<: *pr-refs - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: check-rustdoc - artifacts: false - variables: - RUNTIME_PATH: "parachains/runtimes/assets" - script: - - cd ${RUNTIME_PATH} - - for directory in $(echo */); do - echo "_____Running cargo check for ${directory} ______"; - cd ${directory}; - pwd; - SKIP_WASM_BUILD=1 cargo check; - cd ..; - done - -# DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts -# DAG: build-runtime-assets ->build-runtime-starters -> build-runtime-testing -build-runtime-assets: - <<: *build-runtime-template - variables: - RUNTIME_PATH: "parachains/runtimes/assets" - -build-runtime-collectives: - <<: *build-runtime-template - variables: - RUNTIME_PATH: "parachains/runtimes/collectives" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-runtime-assets - artifacts: false - -build-runtime-contracts: - <<: *build-runtime-template - variables: - RUNTIME_PATH: "parachains/runtimes/contracts" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-runtime-collectives - artifacts: false - -build-runtime-starters: - <<: *build-runtime-template - variables: - RUNTIME_PATH: "parachains/runtimes/starters" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-runtime-assets - artifacts: false - -build-runtime-testing: - <<: *build-runtime-template - variables: - RUNTIME_PATH: "parachains/runtimes/testing" - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-runtime-starters - artifacts: false - - -#### stage: publish - -.build-push-image: &build-push-image - image: quay.io/buildah/stable - variables: - DOCKERFILE: "" # docker/path-to.Dockerfile - IMAGE_NAME: "" # docker.io/paritypr/image_name - VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - script: - - test "$PARITYPR_USER" -a "$PARITYPR_PASS" || - ( echo "no docker credentials provided"; exit 1 ) - - buildah bud - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --build-arg IMAGE_NAME="${IMAGE_NAME}" - --tag "$IMAGE_NAME:$VERSION" - --file ${DOCKERFILE} . - - echo "$PARITYPR_PASS" | - buildah login --username "$PARITYPR_USER" --password-stdin docker.io - - buildah info - - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - after_script: - - buildah logout --all - -build-push-image-polkadot-parachain-debug: - stage: publish - <<: *kubernetes-env - <<: *common-refs - <<: *build-push-image - needs: - - job: build-linux-stable - artifacts: true - variables: - DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" - IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug" - VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - -build-push-image-test-parachain: - stage: publish - <<: *kubernetes-env - <<: *common-refs - <<: *build-push-image - needs: - - job: build-test-parachain - artifacts: true - variables: - DOCKERFILE: "docker/test-parachain_injected.Dockerfile" - IMAGE_NAME: "docker.io/paritypr/test-parachain" - VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - -publish-s3: - stage: publish - <<: *kubernetes-env - image: paritytech/awscli:latest - <<: *publish-refs - needs: - - job: build-linux-stable - artifacts: true - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "cumulus/${ARCH}-${DOCKER_OS}" - script: - - echo "___Publishing a binary with debug assertions!___" - - echo "___VERSION = $(cat ./artifacts/VERSION) ___" - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ - - echo "___Updating objects in latest path___" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - -#### stage: benchmarks -# Work only on release-parachains-v* branches - -benchmarks-build: - stage: benchmarks-build - <<: *docker-env - <<: *collect-artifacts - <<: *benchmarks-manual-refs - script: - - time cargo build --profile production --locked --features runtime-benchmarks - - mkdir -p artifacts - - cp target/production/polkadot-parachain ./artifacts/ - -.git-commit-push: &git-commit-push - - git status - # Set git config - - rm -rf .git/config - - git config --global user.email "${GITHUB_EMAIL}" - - git config --global user.name "${GITHUB_USER}" - - git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - # push results to github - - git checkout -b $BRANCHNAME - - git add parachains/* - - git commit -m "[benchmarks] pr with weights" - - git push origin $BRANCHNAME - -benchmarks-assets: - stage: benchmarks-run - before_script: - - *rust-info-script - <<: *collect-artifacts - <<: *benchmarks-refs - timeout: 1d - script: - - ./scripts/benchmarks-ci.sh assets statemine ./artifacts - - ./scripts/benchmarks-ci.sh assets statemint ./artifacts - - ./scripts/benchmarks-ci.sh assets westmint ./artifacts - - export CURRENT_TIME=$(date '+%s') - - export BRANCHNAME="weights-statemint-${CI_COMMIT_BRANCH}-${CURRENT_TIME}" - - *git-commit-push - # create PR to release-parachains-v* branch - - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} - -d '{"title":"[benchmarks] Update weights for statemine/t","body":"This PR is generated automatically by CI. (Once merged please backport to master and node release branch.)","head":"'${BRANCHNAME}'","base":"'${CI_COMMIT_BRANCH}'"}' - -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls - after_script: - - rm -rf .git/config - tags: - - weights - -benchmarks-collectives: - stage: benchmarks-run - before_script: - - *rust-info-script - <<: *collect-artifacts - <<: *benchmarks-refs +.git-commit-push: script: - - ./scripts/benchmarks-ci.sh collectives collectives-polkadot ./artifacts - git status - - export CURRENT_TIME=$(date '+%s') - - export BRANCHNAME="weights-collectives-${CI_COMMIT_BRANCH}-${CURRENT_TIME}" - - *git-commit-push - # create PR - - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} - -d '{"title":"[benchmarks] Update weights for collectives","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}' - -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls - after_script: + # Set git config - rm -rf .git/config - tags: - - weights - -publish-benchmarks-assets-s3: &publish-benchmarks - stage: publish - <<: *kubernetes-env - image: paritytech/awscli:latest - <<: *benchmarks-refs - needs: - - job: benchmarks-assets - artifacts: true - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks" - script: - - echo "___Removing binary from artifacts___" - - rm -f ./artifacts/polkadot-parachain - - echo "___Publishing benchmark results___" - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize - -publish-benchmarks-collectives-s3: - <<: *publish-benchmarks - needs: - - job: benchmarks-collectives - artifacts: true - -# Ref: https://github.com/paritytech/opstooling/issues/111 -update-parachain-template: - stage: publish - extends: .kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1 - script: - - git clone --depth=1 --branch="$PIPELINE_SCRIPTS_TAG" https://github.com/paritytech/pipeline-scripts - - ./pipeline-scripts/update_substrate_template.sh - --repo-name "substrate-parachain-template" - --template-path "parachain-template" - --github-api-token "$GITHUB_TOKEN" - --polkadot-branch "$CI_COMMIT_REF_NAME" - -#### stage: integration-test - -zombienet-0001-sync_blocks_from_tip_without_connected_collator: - stage: integration-test - image: "${ZOMBIENET_IMAGE}" - <<: *zombienet-refs - needs: - - job: build-push-image-test-parachain - variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" - GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" - COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${RELAY_IMAGE}" - - echo "${COL_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie - - export RELAY_IMAGE=${POLKADOT_IMAGE} - - export COL_IMAGE=${COL_IMAGE} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --concurrency=1 - --test="0001-sync_blocks_from_tip_without_connected_collator.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-0002-pov_recovery: - stage: integration-test - image: "${ZOMBIENET_IMAGE}" - <<: *zombienet-refs - needs: - - job: build-push-image-test-parachain - variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" - GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" - COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${RELAY_IMAGE}" - - echo "${COL_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie - - export RELAY_IMAGE=${POLKADOT_IMAGE} - - export COL_IMAGE=${COL_IMAGE} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --concurrency=1 - --test="0002-pov_recovery.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-0003-full_node_catching_up: - stage: integration-test - image: "${ZOMBIENET_IMAGE}" - <<: *zombienet-refs - needs: - - job: build-push-image-test-parachain - variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" - GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" - COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${RELAY_IMAGE}" - - echo "${COL_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie - - export RELAY_IMAGE=${POLKADOT_IMAGE} - - export COL_IMAGE=${COL_IMAGE} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --concurrency=1 - --test="0003-full_node_catching_up.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-0004-runtime_upgrade: - stage: integration-test - image: "${ZOMBIENET_IMAGE}" - <<: *zombienet-refs - needs: - - job: build-push-image-test-parachain - - job: build-test-parachain - artifacts: true - variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" - GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" - COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - before_script: - - ls -ltr * - - cp ./artifacts/zombienet/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /tmp/ - - ls /tmp - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${RELAY_IMAGE}" - - echo "${COL_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie - - export RELAY_IMAGE=${POLKADOT_IMAGE} - - export COL_IMAGE=${COL_IMAGE} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --concurrency=1 - --test="0004-runtime_upgrade.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-0005-migrate_solo_to_para: - stage: integration-test - image: "${ZOMBIENET_IMAGE}" - <<: *zombienet-refs - needs: - - job: build-push-image-test-parachain - - job: build-test-parachain - artifacts: true - variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" - GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" - COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - before_script: - - ls -ltr * - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${RELAY_IMAGE}" - - echo "${COL_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie - - export RELAY_IMAGE=${POLKADOT_IMAGE} - - export COL_IMAGE=${COL_IMAGE} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --concurrency=1 - --test="0005-migrate_solo_to_para.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test + - git config --global user.email "${GITHUB_EMAIL}" + - git config --global user.name "${GITHUB_USER}" + - git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + # push results to github + - git checkout -b $BRANCHNAME + - git add parachains/* + - git commit -m "[benchmarks] pr with weights" + - git push origin $BRANCHNAME + +include: + # test jobs + - scripts/ci/gitlab/pipeline/test.yml + # # build jobs + - scripts/ci/gitlab/pipeline/build.yml + # # benchmarks jobs + - scripts/ci/gitlab/pipeline/benchmarks.yml + # # publish jobs + - scripts/ci/gitlab/pipeline/publish.yml + # zombienet jobs + - scripts/ci/gitlab/pipeline/zombienet.yml 0006-rpc_collator_builds_blocks: stage: integration-test @@ -669,7 +172,7 @@ zombienet-0005-migrate_solo_to_para: # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. -.cancel-pipeline: +cancel-pipeline: stage: .post needs: - job: test-linux-stable diff --git a/cumulus/scripts/ci/gitlab/pipeline/benchmarks.yml b/cumulus/scripts/ci/gitlab/pipeline/benchmarks.yml new file mode 100644 index 00000000000..03aedd03960 --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/benchmarks.yml @@ -0,0 +1,61 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "benchmarks" stage +# Work only on release-parachains-v* branches + +benchmarks-build: + stage: benchmarks-build + extends: + - .docker-env + - .collect-artifacts + - .benchmarks-manual-refs + script: + - time cargo build --profile production --locked --features runtime-benchmarks + - mkdir -p artifacts + - cp target/production/polkadot-parachain ./artifacts/ + +benchmarks-assets: + stage: benchmarks-run + extends: + - .collect-artifacts + - .benchmarks-refs + before_script: + - !reference [.docker-env, before_script] + timeout: 1d + script: + - ./scripts/benchmarks-ci.sh assets statemine ./artifacts + - ./scripts/benchmarks-ci.sh assets statemint ./artifacts + - ./scripts/benchmarks-ci.sh assets westmint ./artifacts + - export CURRENT_TIME=$(date '+%s') + - export BRANCHNAME="weights-statemint-${CI_COMMIT_BRANCH}-${CURRENT_TIME}" + - !reference [.git-commit-push, script] + # create PR to release-parachains-v* branch + - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} + -d '{"title":"[benchmarks] Update weights for statemine/t","body":"This PR is generated automatically by CI. (Once merged please backport to master and node release branch.)","head":"'${BRANCHNAME}'","base":"'${CI_COMMIT_BRANCH}'"}' + -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls + after_script: + - rm -rf .git/config + tags: + - weights + +benchmarks-collectives: + stage: benchmarks-run + extends: + - .collect-artifacts + - .benchmarks-refs + before_script: + - !reference [.docker-env, before_script] + script: + - ./scripts/benchmarks-ci.sh collectives collectives-polkadot ./artifacts + - git status + - export CURRENT_TIME=$(date '+%s') + - export BRANCHNAME="weights-collectives-${CI_COMMIT_BRANCH}-${CURRENT_TIME}" + - !reference [.git-commit-push, script] + # create PR + - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} + -d '{"title":"[benchmarks] Update weights for collectives","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}' + -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls + after_script: + - rm -rf .git/config + tags: + - weights + diff --git a/cumulus/scripts/ci/gitlab/pipeline/build.yml b/cumulus/scripts/ci/gitlab/pipeline/build.yml new file mode 100644 index 00000000000..86081e6a46a --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/build.yml @@ -0,0 +1,113 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "build" stage + +build-linux-stable: + stage: build + extends: + - .docker-env + - .common-refs + - .collect-artifacts + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-rustdoc + artifacts: false + script: + - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" + - time cargo build --release --locked --bin polkadot-parachain + - echo "___Packing the artifacts___" + - mkdir -p ./artifacts + - mv ./target/release/polkadot-parachain ./artifacts/. + - echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___" + - echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION + +build-test-parachain: + stage: build + extends: + - .docker-env + - .common-refs + - .collect-artifacts + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-rustdoc + artifacts: false + script: + - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" + - time cargo +nightly build --release --locked --bin test-parachain + - echo "___Packing the artifacts___" + - mkdir -p ./artifacts + - mv ./target/release/test-parachain ./artifacts/. + - mkdir -p ./artifacts/zombienet + - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/. + +# build runtime only if files in $RUNTIME_PATH/$RUNTIME_NAME were changed +.build-runtime-template: &build-runtime-template + stage: build + extends: + - .docker-env + - .pr-refs + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-rustdoc + artifacts: false + variables: + RUNTIME_PATH: "parachains/runtimes/assets" + script: + - cd ${RUNTIME_PATH} + - for directory in $(echo */); do + echo "_____Running cargo check for ${directory} ______"; + cd ${directory}; + pwd; + SKIP_WASM_BUILD=1 cargo check; + cd ..; + done + +# DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts +# DAG: build-runtime-assets ->build-runtime-starters -> build-runtime-testing +build-runtime-assets: + <<: *build-runtime-template + variables: + RUNTIME_PATH: "parachains/runtimes/assets" + +build-runtime-collectives: + <<: *build-runtime-template + variables: + RUNTIME_PATH: "parachains/runtimes/collectives" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-runtime-assets + artifacts: false + +build-runtime-contracts: + <<: *build-runtime-template + variables: + RUNTIME_PATH: "parachains/runtimes/contracts" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-runtime-collectives + artifacts: false + +build-runtime-starters: + <<: *build-runtime-template + variables: + RUNTIME_PATH: "parachains/runtimes/starters" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-runtime-assets + artifacts: false + +build-runtime-testing: + <<: *build-runtime-template + variables: + RUNTIME_PATH: "parachains/runtimes/testing" + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-runtime-starters + artifacts: false diff --git a/cumulus/scripts/ci/gitlab/pipeline/integration_tests.yml b/cumulus/scripts/ci/gitlab/pipeline/integration_tests.yml new file mode 100644 index 00000000000..a884361aa7c --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/integration_tests.yml @@ -0,0 +1,2 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "integration_stage" stage diff --git a/cumulus/scripts/ci/gitlab/pipeline/publish.yml b/cumulus/scripts/ci/gitlab/pipeline/publish.yml new file mode 100644 index 00000000000..5773bc2b066 --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/publish.yml @@ -0,0 +1,117 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "publish" stage + +.build-push-image: + image: quay.io/buildah/stable + variables: + DOCKERFILE: "" # docker/path-to.Dockerfile + IMAGE_NAME: "" # docker.io/paritypr/image_name + VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + script: + - test "$PARITYPR_USER" -a "$PARITYPR_PASS" || + ( echo "no docker credentials provided"; exit 1 ) + - buildah bud + --format=docker + --build-arg VCS_REF="${CI_COMMIT_SHA}" + --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + --build-arg IMAGE_NAME="${IMAGE_NAME}" + --tag "$IMAGE_NAME:$VERSION" + --file ${DOCKERFILE} . + - echo "$PARITYPR_PASS" | + buildah login --username "$PARITYPR_USER" --password-stdin docker.io + - buildah info + - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" + after_script: + - buildah logout --all + +build-push-image-polkadot-parachain-debug: + stage: publish + extends: + - .kubernetes-env + - .common-refs + - .build-push-image + needs: + - job: build-linux-stable + artifacts: true + variables: + DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" + IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug" + VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + +build-push-image-test-parachain: + stage: publish + extends: + - .kubernetes-env + - .common-refs + - .build-push-image + needs: + - job: build-test-parachain + artifacts: true + variables: + DOCKERFILE: "docker/test-parachain_injected.Dockerfile" + IMAGE_NAME: "docker.io/paritypr/test-parachain" + VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + +publish-s3: + stage: publish + extends: + - .kubernetes-env + - .publish-refs + image: paritytech/awscli:latest + needs: + - job: build-linux-stable + artifacts: true + variables: + GIT_STRATEGY: none + BUCKET: "releases.parity.io" + PREFIX: "cumulus/${ARCH}-${DOCKER_OS}" + script: + - echo "___Publishing a binary with debug assertions!___" + - echo "___VERSION = $(cat ./artifacts/VERSION) ___" + - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ + - echo "___Updating objects in latest path___" + - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ + after_script: + - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ + --recursive --human-readable --summarize + +publish-benchmarks-assets-s3: &publish-benchmarks + stage: publish + extends: + - .kubernetes-env + - .benchmarks-refs + image: paritytech/awscli:latest + needs: + - job: benchmarks-assets + artifacts: true + variables: + GIT_STRATEGY: none + BUCKET: "releases.parity.io" + PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks" + script: + - echo "___Removing binary from artifacts___" + - rm -f ./artifacts/polkadot-parachain + - echo "___Publishing benchmark results___" + - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/ + after_script: + - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize + +publish-benchmarks-collectives-s3: + <<: *publish-benchmarks + needs: + - job: benchmarks-collectives + artifacts: true + +# Ref: https://github.com/paritytech/opstooling/issues/111 +update-parachain-template: + stage: publish + extends: .kubernetes-env + rules: + - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1 + script: + - git clone --depth=1 --branch="$PIPELINE_SCRIPTS_TAG" https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/update_substrate_template.sh + --repo-name "substrate-parachain-template" + --template-path "parachain-template" + --github-api-token "$GITHUB_TOKEN" + --polkadot-branch "$CI_COMMIT_REF_NAME" diff --git a/cumulus/scripts/ci/gitlab/pipeline/test.yml b/cumulus/scripts/ci/gitlab/pipeline/test.yml new file mode 100644 index 00000000000..0db2bb554b0 --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/test.yml @@ -0,0 +1,79 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "test" stage + +test-linux-stable: + stage: test + extends: + - .docker-env + - .common-refs + - .pipeline-stopper-artifacts + before_script: + - !reference [.docker-env, before_script] + - !reference [.pipeline-stopper-vars, before_script] + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo nextest run --all --release --locked --run-ignored all + +test-doc: + stage: test + extends: + - .docker-env + - .common-refs + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --doc + +check-runtime-benchmarks: + stage: test + extends: + - .docker-env + - .common-refs + script: + # Check that the node will compile with `runtime-benchmarks` feature flag. + - time cargo check --all --features runtime-benchmarks + # Check that parachain-template will compile with `runtime-benchmarks` feature flag. + - time cargo check -p parachain-template-node --features runtime-benchmarks + +cargo-check-try-runtime: + stage: test + extends: + - .docker-env + - .common-refs + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-runtime-benchmarks + artifacts: false + script: + # Check that the node will compile with `try-runtime` feature flag. + - time cargo check --all --features try-runtime + # Check that parachain-template will compile with `try-runtime` feature flag. + - time cargo check -p parachain-template-node --features try-runtime + +check-rustdoc: + stage: test + extends: + - .docker-env + - .common-refs + variables: + SKIP_WASM_BUILD: 1 + RUSTDOCFLAGS: "-Dwarnings" + script: + - time cargo +nightly doc --workspace --all-features --verbose --no-deps + +cargo-check-benches: + stage: test + extends: + - .docker-env + - .common-refs + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: check-rustdoc + artifacts: false + script: + - time cargo check --all --benches diff --git a/cumulus/scripts/ci/gitlab/pipeline/zombienet.yml b/cumulus/scripts/ci/gitlab/pipeline/zombienet.yml new file mode 100644 index 00000000000..fcd0a9b669d --- /dev/null +++ b/cumulus/scripts/ci/gitlab/pipeline/zombienet.yml @@ -0,0 +1,101 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "zombienet" stage + +.zombienet-before-script: + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${RELAY_IMAGE}" + - echo "${COL_IMAGE}" + - echo "${GH_DIR}" + - export DEBUG=zombie + - export RELAY_IMAGE=${POLKADOT_IMAGE} + - export COL_IMAGE=${COL_IMAGE} + +# common settings for all zombienet jobs +.zombienet-common: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + needs: + - job: build-push-image-test-parachain + artifacts: true + variables: + POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" + GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" + COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-0001-sync_blocks_from_tip_without_connected_collator: + extends: + - .zombienet-common + - .zombienet-refs + - .zombienet-before-script + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --concurrency=1 + --test="0001-sync_blocks_from_tip_without_connected_collator.feature" + +zombienet-0002-pov_recovery: + extends: + - .zombienet-common + - .zombienet-refs + - .zombienet-before-script + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --concurrency=1 + --test="0002-pov_recovery.feature" + +zombienet-0003-full_node_catching_up: + extends: + - .zombienet-common + - .zombienet-refs + - .zombienet-before-script + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --concurrency=1 + --test="0003-full_node_catching_up.feature" + +zombienet-0004-runtime_upgrade: + extends: + - .zombienet-common + - .zombienet-refs + - .zombienet-before-script + needs: + - !reference [.zombienet-common, needs] + - job: build-test-parachain + artifacts: true + before_script: + - ls -ltr * + - cp ./artifacts/zombienet/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /tmp/ + - ls /tmp + - !reference [.zombienet-before-script, before_script] + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --concurrency=1 + --test="0004-runtime_upgrade.feature" + +zombienet-0005-migrate_solo_to_para: + extends: + - .zombienet-common + - .zombienet-refs + - .zombienet-before-script + needs: + - !reference [.zombienet-common, needs] + - job: build-test-parachain + artifacts: true + before_script: + - ls -ltr * + - !reference [.zombienet-before-script, before_script] + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --concurrency=1 + --test="0005-migrate_solo_to_para.feature" + -- GitLab