Skip to content
Snippets Groups Projects
Commit 82e52b8b authored by Denis_P's avatar Denis_P :field_hockey: Committed by GitHub
Browse files

CI: test and update ci image, codeowners (#8142) 

* CI: prep for the new ff deployment

* CI: variable for CI image

* git: add CI team and remove Max from CODEOWNERS

* CI: diener should be updated in CI image, not here.

* CI: diener should be updated in CI image, not here.

* CI: run cargo deny on changes to manifests and lock; run build jobs on schedules [skip ci]

* CI: remove flaming-fir deployment, it will be handled from s3 updates [skip ci]

* CI: trigger simnet with a certain substrate version

* CI: remove cargo-audit in favor of cargo-deny; prepare for being triggered

* CI: prepare to be triggered

* CI: chore
parent 20527aec
No related merge requests found
Hide whitespace changes
Inline Side-by-side
substrate/.gitlab-ci.yml
+ 57
− 84
View file @ 82e52b8b
......@@ -39,6 +39,7 @@ variables: &default-vars
CARGO_INCREMENTAL: 0
DOCKER_OS: "debian:stretch"
ARCH: "x86_64"
CI_IMAGE: "paritytech/ci-linux:production"
# FIXME set to release
CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.11"
CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder"
......@@ -60,7 +61,7 @@ default:
interruptible: true
.docker-env: &docker-env
image: paritytech/ci-linux:production
image: "${CI_IMAGE}"
before_script:
- rustup show
- cargo --version
......@@ -84,12 +85,39 @@ default:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
.test-refs-no-trigger: &test-refs-no-trigger
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
.build-refs: &build-refs
rules:
# .publish-refs with manual on PRs
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: manual
allow_failure: true
.publish-refs: &publish-refs
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
#### stage: .pre
......@@ -145,6 +173,8 @@ test-dependency-rules:
stage: check
image: paritytech/tools:latest
<<: *kubernetes-build
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
script:
- .maintain/ensure-deps.sh
......@@ -153,36 +183,28 @@ test-prometheus-alerting-rules:
image: paritytech/tools:latest
<<: *kubernetes-build
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_BRANCH
changes:
- .gitlab-ci.yml
- .maintain/monitoring/**/*
script:
- promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
- cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
- cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
#### stage: test
cargo-audit:
stage: test
<<: *docker-env
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: never
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- cargo audit
allow_failure: true
cargo-deny:
stage: test
<<: *docker-env
rules:
- if: $CI_COMMIT_MESSAGE =~ /skip-checks/
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- changes:
- "Cargo.lock"
- "**/Cargo.toml"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
......@@ -205,7 +227,7 @@ cargo-deny:
cargo-check-benches:
stage: test
<<: *docker-env
<<: *test-refs
<<: *test-refs-no-trigger
script:
- SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
- cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small
......@@ -262,9 +284,8 @@ unleash-check:
stage: test
<<: *docker-env
rules:
- if: $CI_COMMIT_MESSAGE =~ /skip-checks/
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
# .test-refs
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
......@@ -281,7 +302,7 @@ test-frame-examples-compile-to-wasm:
<<: *default-vars
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: -Cdebug-assertions=y
RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1
script:
- cd frame/example-offchain-worker/
......@@ -333,7 +354,7 @@ test-full-crypto-feature:
<<: *default-vars
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: -Cdebug-assertions=y
RUSTFLAGS: "-Cdebug-assertions=y"
RUST_BACKTRACE: 1
script:
- cd primitives/core/
......@@ -346,7 +367,7 @@ cargo-check-macos:
stage: test
# shell runner on mac ignores the image set in *docker-env
<<: *docker-env
<<: *test-refs
<<: *test-refs-no-trigger
script:
- SKIP_WASM_BUILD=1 time cargo check --release
- sccache -s
......@@ -367,7 +388,7 @@ check-polkadot-companion-status:
check-polkadot-companion-build:
stage: build
<<: *docker-env
<<: *test-refs
<<: *test-refs-no-trigger
needs:
- job: test-linux-stable-int
artifacts: false
......@@ -396,15 +417,7 @@ build-linux-substrate: &build-binary
stage: build
<<: *collect-artifacts
<<: *docker-env
rules:
# .build-refs with manual on PRs
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: manual
allow_failure: true
<<: *build-refs
needs:
- job: test-linux-stable
artifacts: false
......@@ -431,15 +444,7 @@ build-linux-subkey: &build-subkey
stage: build
<<: *collect-artifacts
<<: *docker-env
rules:
# .build-refs with manual on PRs
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "tags"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: manual
allow_failure: true
<<: *build-refs
needs:
- job: cargo-check-subkey
artifacts: false
......@@ -466,7 +471,7 @@ build-macos-subkey:
#### stage: publish
.build-push-docker-image: &build-push-docker-image
<<: *build-refs
<<: *publish-refs
<<: *kubernetes-build
image: quay.io/buildah/stable
variables: &docker-build-vars
......@@ -499,8 +504,6 @@ build-macos-subkey:
publish-docker-substrate:
stage: publish
<<: *build-push-docker-image
# collect VERSION artifact here to pass it on to kubernetes
<<: *collect-artifacts
needs:
- job: build-linux-substrate
artifacts: true
......@@ -508,8 +511,12 @@ publish-docker-substrate:
<<: *docker-build-vars
PRODUCT: substrate
after_script:
# only VERSION information is needed for the deployment
- find ./artifacts/ -depth -not -name VERSION -type f -delete
- echo "VERSION=${VERSION}" >> build.env
artifacts:
reports:
# this artifact is used in trigger-simnet job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
dotenv: artifacts/substrate/build.env
publish-docker-subkey:
stage: publish
......@@ -523,7 +530,7 @@ publish-docker-subkey:
publish-s3-release:
stage: publish
<<: *build-refs
<<: *publish-refs
<<: *kubernetes-build
needs:
- job: build-linux-substrate
......@@ -587,6 +594,8 @@ deploy-prometheus-alerting-rules:
- kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
--type=merge --patch "$(sed 's/^/ /;1s/^/spec:\n/' ${RULES})"
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_REF_NAME == "master"
changes:
- .gitlab-ci.yml
......@@ -599,43 +608,7 @@ trigger-simnet:
- if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
needs:
- job: publish-docker-substrate
artifacts: false
trigger:
project: parity/simnet
branch: master
strategy: depend
.validator-deploy: &validator-deploy
stage: deploy
rules:
# this job runs only on nightly pipeline with the mentioned variable, against `master` branch
- if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
needs:
# script will fail if there is no artifacts/substrate/VERSION
- job: publish-docker-substrate
artifacts: true
image: parity/azure-ansible:v2
allow_failure: true
interruptible: true
tags:
- linux-docker
validator 1 4:
<<: *validator-deploy
script:
- ./.maintain/flamingfir-deploy.sh flamingfir-validator1
validator 2 4:
<<: *validator-deploy
script:
- ./.maintain/flamingfir-deploy.sh flamingfir-validator2
validator 3 4:
<<: *validator-deploy
script:
- ./.maintain/flamingfir-deploy.sh flamingfir-validator3
validator 4 4:
<<: *validator-deploy
script:
- ./.maintain/flamingfir-deploy.sh flamingfir-validator4
substrate/.maintain/flamingfir-deploy.sh deleted 100755 → 0
+ 0
− 35
View file @ 20527aec
#!/bin/bash
RETRY_COUNT=10
RETRY_ATTEMPT=0
SLEEP_TIME=15
TARGET_HOST="$1"
COMMIT=$(cat artifacts/substrate/VERSION)
DOWNLOAD_URL="https://releases.parity.io/substrate/x86_64-debian:stretch/${COMMIT}/substrate/substrate"
POST_DATA='{"extra_vars":{"artifact_path":"'${DOWNLOAD_URL}'","target_host":"'${TARGET_HOST}'"}}'
JOB_ID=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" --header "Content-type: application/json" --post-data "${POST_DATA}" https://ansible-awx.parity.io/api/v2/job_templates/32/launch/ | jq .job)
echo "Launched job: $JOB_ID"
while [ ${RETRY_ATTEMPT} -le ${RETRY_COUNT} ] ; do
export RETRY_RESULT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status)
RETRY_ATTEMPT=$(( $RETRY_ATTEMPT +1 ))
sleep $SLEEP_TIME
if [ $(echo $RETRY_RESULT | egrep -e successful -e failed) ] ; then
break
fi
done
AWX_OUTPUT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/stdout?format=txt_download)
echo "AWX job log:"
echo "${AWX_OUTPUT}"
JOB_STATUS=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status )
echo "==================================="
echo -e "Ansible AWX Remote Job: ${JOB_ID} \x1B[31mStatus: ${JOB_STATUS}\x1B[0m"
echo "==================================="
substrate/.maintain/gitlab/check_polkadot_companion_build.sh
+ 0
− 2
View file @ 82e52b8b
......@@ -51,8 +51,6 @@ git merge origin/master
# ancestor for successfully performing merges below.
git clone --depth 20 https://github.com/paritytech/polkadot.git
cargo install -f diener
cd polkadot
# either it's a pull request then check for a companion otherwise use
......
substrate/docs/CODEOWNERS
+ 5
− 7
View file @ 82e52b8b
......@@ -18,6 +18,11 @@
# are more recognizable on GitHub, you can use them for mentioning unlike an email.
# - The latest matching rule, if multiple, takes precedence.
# CI
/.maintain/ @paritytech/ci
/.github/ @paritytech/ci
/.gitlab-ci.yml @paritytech/ci
# Block production
/client/basic-authorship/ @NikVolf
......@@ -56,10 +61,3 @@
# Transaction weight stuff
/frame/support/src/weights.rs @shawntabrizi
# Authority discovery
/client/authority-discovery/ @mxinden
/frame/authority-discovery/ @mxinden
# Prometheus endpoint
/utils/prometheus/ @mxinden
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment