Newer
Older
gabriel klawitter
committed
# .gitlab-ci.yml
#
gabriel klawitter
committed
# pipelines can be triggered manually in the web
# setting DEPLOY_TAG will only deploy the tagged image
#
# please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner
gabriel klawitter
committed
gabriel klawitter
committed
- publish
- deploy
workflow:
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io"
VAULT_AUTH_PATH: "gitlab-parity-io-jwt"
VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}"
SIMNET_IMAGE: "europe-west3-docker.pkg.dev/parity-simnet/simnet-images/simnet:v14"
PIPELINE_SCRIPTS_TAG: "v0.1"
gabriel klawitter
committed
.collect-artifacts: &collect-artifacts
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: on_success
.kubernetes-env: &kubernetes-env
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
interruptible: true
tags:
- kubernetes-parity-build
.docker-env: &docker-env
when:
- runner_system_failure
- unknown_failure
- api_failure
tags:
- linux-docker
.compiler-info: &compiler-info
before_script:
- rustup show
- cargo --version
- sccache -s
.rules-test: &rules-test
# these jobs run always*
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- when: always
.pr-only: &rules-pr-only
# these jobs run only on PRs
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
#### Vault secrets
.vault-secrets: &vault-secrets
secrets:
AWS_ACCESS_KEY_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv
file: false
AWS_SECRET_ACCESS_KEY:
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv
file: false
DOCKER_HUB_USER:
vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv
file: false
DOCKER_HUB_PASS:
vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv
file: false
GITHUB_PR_TOKEN:
vault: cicd/gitlab/parity/GITHUB_PR_TOKEN@kv
file: false
GITHUB_USER:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv
file: false
GITHUB_RELEASE_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv
file: false
GITHUB_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv
MATRIX_ACCESS_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv
file: false
MATRIX_ROOM_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv
PARITYPR_USER:
vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_USER@kv
file: false
PARITYPR_PASS:
vault: cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_PASS@kv
file: false
PIPELINE_TOKEN:
vault: cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv
file: false
REL_MAN_ROOM_ID:
vault: cicd/gitlab/$CI_PROJECT_PATH/REL_MAN_ROOM_ID@kv
file: false
SSH_PRIVATE_KEY:
vault: cicd/gitlab/$CI_PROJECT_PATH/SSH_PRIVATE_KEY@kv
file: false
#### stage: test
check-runtime:
stage: test
variables:
GITLAB_API: "https://gitlab.parity.io/api/v4"
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
script:
- ./scripts/gitlab/check_runtime.sh
allow_failure: true
check-line-width:
stage: test
script:
- ./scripts/gitlab/check_line_width.sh
allow_failure: true
cargo-fmt:
stage: test
<<: *kubernetes-env
<<: *rules-pr-only
script:
- cargo +nightly fmt --all -- --check
allow_failure: true
test-deterministic-wasm:
stage: test
<<: *docker-env
script:
- ./scripts/gitlab/test_deterministic_wasm.sh
<<: *docker-env
variables:
RUST_TOOLCHAIN: stable
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
gabriel klawitter
committed
script:
- ./scripts/gitlab/test_linux_stable.sh
# we're using the bin built here, instead of having a parallel `build-linux-release`
- time cargo build --release --verbose --bin polkadot --features "disputes"
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
- EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
script:
# Check that the node will compile with `runtime-benchmarks` feature flag.
- ./scripts/gitlab/check_runtime_benchmarks.sh
check-no-default-features:
stage: test
<<: *rules-test
<<: *docker-env
<<: *compiler-info
<<: *vault-secrets
script:
# Check that polkadot-cli will compile no default features.
- ./scripts/gitlab/check_no_default_features.sh
- sccache -s
spellcheck:
stage: test
<<: *docker-env
<<: *rules-pr-only
script:
- cargo spellcheck --version
# compare with the commit parent to the PR, given it's from a default branch
- git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
- cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
- time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
$(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
build-adder-collator:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
script:
- time cargo build --release --verbose -p test-parachain-adder-collator
- sccache -s
# pack artifacts
- mkdir -p ./artifacts
- mv ./target/release/adder-collator ./artifacts/.
- echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
- echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
- echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
.check-dependent-project: &check-dependent-project
stage: build
<<: *docker-env
<<: *vault-secrets
script:
- git clone
--depth=1
"--branch=$PIPELINE_SCRIPTS_TAG"
https://github.com/paritytech/pipeline-scripts
- ./pipeline-scripts/check_dependent_project.sh
paritytech
polkadot
--polkadot
"$DEPENDENT_REPO"
"$GITHUB_PR_TOKEN"
check-dependent-cumulus:
<<: *check-dependent-project
variables:
DEPENDENT_REPO: cumulus
# image must be ubuntu:20.04 based to match the linkers, this image has npm installed
image: paritytech/contracts-ci-linux:production
stage: build
<<: *rules-test
<<: *docker-env
<<: *vault-secrets
needs:
- job: test-build-linux-stable
artifacts: true
before_script:
- apt-get -y update; apt-get -y install jq lsof
- npm install --ignore-scripts -g @polkadot/metadata-cmp
- git fetch origin release
script:
- scripts/gitlab/check_extrinsics_ordering.sh
stage: build
<<: *rules-test
<<: *docker-env
name: michaelfbryan/mdbook-docker-image:v0.4.4
- mdbook build roadmap/implementers-guide
build-rustdoc:
stage: build
<<: *docker-env
<<: *rules-test
variables:
SKIP_WASM_BUILD: 1
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
when: on_success
expire_in: 7 days
paths:
- ./crate-docs/
script:
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
- time cargo doc --no-deps --workspace --verbose
- rm -f ./target/doc/.lock
- mv ./target/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot
- chown -R nonroot:nonroot ./crate-docs
- echo "<meta http-equiv=refresh content=0;url=polkadot_service/index.html>" > ./crate-docs/index.html
- sccache -s
<<: *kubernetes-env
image: quay.io/buildah/stable
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
before_script: &check-versions
- test -s ./artifacts/VERSION || exit 1
- test -s ./artifacts/EXTRATAG || exit 1
- VERSION="$(cat ./artifacts/VERSION)"
- EXTRATAG="$(cat ./artifacts/EXTRATAG)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- test "$DOCKER_USER" -a "$DOCKER_PASS" ||
( echo "no docker credentials provided"; exit 1 )
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--tag "$IMAGE_NAME:$EXTRATAG"
--file ${DOCKERFILE} .
# The job will success only on the protected branch
- echo "$DOCKER_PASS" |
buildah login --username "$DOCKER_USER" --password-stdin docker.io
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
- buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
gabriel klawitter
committed
after_script:
variables:
<<: *image-variables
# scripts/dockerfiles/polkadot_injected_debug.Dockerfile
DOCKERFILE: dockerfiles/polkadot_injected_debug.Dockerfile
IMAGE_NAME: docker.io/paritypr/synth-wave
# Don't run on releases - this is handled by the Github Action here:
# .github/workflows/publish-docker-release.yml
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
IMAGE_NAME: docker.io/parity/rococo
DOCKER_USER: ${DOCKER_HUB_USER}
DOCKER_PASS: ${DOCKER_HUB_PASS}
after_script:
# pass artifacts to the simnet-tests job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
- echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
- echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
artifacts:
reports:
# this artifact is used in simnet-tests job
dotenv: ./artifacts/parachains.env
publish-adder-collator-image:
# service image for Simnet
stage: build
<<: *build-push-image
variables:
<<: *image-variables
# scripts/dockerfiles/collator_injected.Dockerfile
DOCKERFILE: dockerfiles/collator_injected.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: build-adder-collator
artifacts: true
after_script:
# pass artifacts to the simnet-tests job
- echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
- echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
dotenv: ./artifacts/collator.env
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
.update_weights: &update-weights
stage: build
when: manual
tags:
- weights
variables:
RUNTIME: polkadot
artifacts:
paths:
- ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
script: |
./scripts/run_benches_for_runtime.sh $RUNTIME
git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
update_polkadot_weights:
<<: *update-weights
update_kusama_weights:
<<: *update-weights
variables:
RUNTIME: kusama
update_westend_weights:
<<: *update-weights
variables:
RUNTIME: westend
artifacts: true
<<: *kubernetes-env
variables:
GIT_STRATEGY: none
- if: $CI_PIPELINE_SOURCE == "schedule"
before_script:
- *check-versions
- echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
- aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
- echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
- find ./artifacts -type f | while read file; do
name="${file#./artifacts/}";
aws s3api copy-object
--copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
--bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
- |
cat <<-EOM
|
| - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
| - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
- aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
--recursive --human-readable --summarize
gabriel klawitter
committed
publish-rustdoc:
stage: publish
<<: *kubernetes-env
<<: *vault-secrets
image: paritytech/tools:latest
variables:
GIT_DEPTH: 100
rules:
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"
# `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
# artifacts from being dowloaded by this job.
needs:
- job: build-rustdoc
artifacts: true
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
script:
- rm -rf /tmp/*
# Set git config
- rm -rf .git/config
- git config user.email "devops-team@parity.io"
- git config user.name "${GITHUB_USER}"
- git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/polkadot.git"
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
- git fetch origin gh-pages
# Save README and docs
- cp -r ./crate-docs/ /tmp/doc/
- cp README.md /tmp/doc/
- git checkout gh-pages
# Remove everything and restore generated docs and README
- rm -rf ./*
- mv /tmp/doc/* .
# Upload files
- git add --all --force
# `git commit` has an exit code of > 0 if there is nothing to commit.
# This causes GitLab to exit immediately and marks this job failed.
# We don't want to mark the entire job failed if there's nothing to
# publish though, hence the `|| true`.
- git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
echo "___Nothing to commit___"
- git push origin gh-pages --force
after_script:
- rm -rf .git/ ./*
deploy-polkasync-kusama:
stage: deploy
rules:
# former .rules-build
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
variables:
gabriel klawitter
committed
POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}"
POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}"
allow_failure: true
gabriel klawitter
committed
trigger: "parity/infrastructure/parity-testnet"
variables:
# FIXME: change branch to "master" after merging this PR
# GH_DIR: "https://github.com/paritytech/polkadot/tree/master/simnet_tests"
GH_DIR: "https://github.com/paritytech/polkadot/tree/simnet-v10/simnet_tests"
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: publish-polkadot-image
- job: publish-adder-collator-image
# `parachains.env` brings here `$PARACHAINS_IMAGE_NAME` and `$PARACHAINS_IMAGE_TAG`
# (`$EXTRATAG` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
# `collator.env` brings here `$COLLATOR_IMAGE_NAME` and `$COLLATOR_IMAGE_TAG`
# For local tests with docker `$PARACHAINS_IMAGE_TAG` and `$COLLATOR_IMAGE_TAG`
# can be replaced with `master` tag.
# SIMNET_REF is a gitlab variable
before_script:
- echo "Simnet Tests Config"
- echo "${SIMNET_IMAGE}"
- echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
- echo "${COLLATOR_IMAGE_NAME} ${COLLATOR_IMAGE_TAG}"
- echo "${GH_DIR}"
- /home/nonroot/simnet/scripts/run-test-environment-manager-v2.sh
--github-remote-dir="${GH_DIR}"
--tag=smoketest
--image="PARACHAINSIMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
--image="SYNTHIMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
--image="COLIMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}"
--image="SCRIPTSIMAGE=${SIMNET_IMAGE}"