Skip to content
Snippets Groups Projects
Commit e5ca9452 authored by João Paulo Silva de Souza's avatar João Paulo Silva de Souza Committed by GitHub
Browse files

use pipeline-scripts for dependent projects script (#9749)

parent e4f92138
Branches
No related merge requests found
......@@ -46,6 +46,7 @@ variables: &default-vars
VAULT_AUTH_PATH: "gitlab-parity-io-jwt"
VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}"
SIMNET_FEATURES_PATH: "simnet_tests/tests"
PIPELINE_SCRIPTS_TAG: "v0.1"
default:
cache: {}
......@@ -538,29 +539,37 @@ cargo-check-macos:
#### stage: build
check-polkadot-companion-status:
stage: build
image: paritytech/tools:latest
<<: *kubernetes-env
<<: *vault-secrets
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
script:
- ./.maintain/gitlab/check_polkadot_companion_status.sh
check-polkadot-companion-build:
.check-dependent-project: &check-dependent-project
stage: build
<<: *docker-env
<<: *test-refs-no-trigger
<<: *vault-secrets
needs:
- job: test-linux-stable-int
artifacts: false
script:
- ./.maintain/gitlab/check_polkadot_companion_build.sh
after_script:
- cd polkadot && git rev-parse --abbrev-ref HEAD
allow_failure: true
- git clone
--depth=1
"--branch=$PIPELINE_SCRIPTS_TAG"
https://github.com/paritytech/pipeline-scripts
- ./pipeline-scripts/check_dependent_project.sh
paritytech
substrate
--substrate
"$DEPENDENT_REPO"
"$GITHUB_PR_TOKEN"
- cd "$DEPENDENT_REPO" && git rev-parse --abbrev-ref HEAD
# Individual jobs are set up for each dependent project so that they can be ran in parallel.
# Arguably we could generate a job for each companion in the PR's description using Gitlab's
# parent-child pipelines but that's more complicated.
check-dependent-polkadot:
<<: *check-dependent-project
variables:
DEPENDENT_REPO: polkadot
check-dependent-cumulus:
<<: *check-dependent-project
variables:
DEPENDENT_REPO: cumulus
build-linux-substrate: &build-binary
stage: build
......@@ -884,7 +893,7 @@ deploy-prometheus-alerting-rules:
# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master
# A "quick" test is a smoke test where basic check-expect tests run by
# checking values from metrics exposed by the app.
# A "long" test is the load testing where we send 50K transactions into the
# A "long" test is the load testing where we send 50K transactions into the
# network and check if all completed successfully
simnet-tests:
stage: deploy
......
#!/usr/bin/env sh
#
# check if a pr is compatible with polkadot companion pr or master if not
# available
#
# to override one that was just mentioned mark companion pr in the body of the
# polkadot pr like
#
# polkadot companion: paritytech/polkadot#567
#
set -e
github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
# use github api v3 in order to access the data without authentication
github_header="Authorization: token ${GITHUB_PR_TOKEN}"
boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
boldcat <<-EOT
check_polkadot_companion_build
==============================
this job checks if there is a string in the description of the pr like
polkadot companion: paritytech/polkadot#567
it will then run cargo check from this polkadot's branch with substrate code
from this pull request. otherwise, it will uses master instead
EOT
# Set the user name and email to make merging work
git config --global user.name 'CI system'
git config --global user.email '<>'
# Merge master into our branch before building Polkadot to make sure we don't miss
# any commits that are required by Polkadot.
git fetch --depth 100 origin
git merge origin/master
# Clone the current Polkadot master branch into ./polkadot.
# NOTE: we need to pull enough commits to be able to find a common
# ancestor for successfully performing merges below.
git clone --depth 20 https://github.com/paritytech/polkadot.git
cd polkadot
# either it's a pull request then check for a companion otherwise use
# polkadot:master
if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null
then
boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
pr_data_file="$(mktemp)"
# get the last reference to a pr in polkadot
curl -sSL -H "${github_header}" -o "${pr_data_file}" \
"${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}"
pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")"
pr_companion="$(echo "${pr_body}" | sed -n -r \
-e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
-e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
| tail -n 1)"
if [ "${pr_companion}" ]
then
boldprint "companion pr specified/detected: #${pr_companion}"
git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion}
git checkout pr/${pr_companion}
git merge origin/master
else
boldprint "no companion branch found - building polkadot:master"
fi
rm -f "${pr_data_file}"
else
boldprint "this is not a pull request - building polkadot:master"
fi
# Patch all Substrate crates in Polkadot
diener patch --crates-to-patch ../ --substrate --path Cargo.toml
# We need to update specifically our patched Substrate crates so that other
# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version
# NOTE: There's no way to only update patched crates, so we use a heuristic
# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of
# updating unrelated dependencies
cargo update -p sp-core
# Test Polkadot pr or master branch with this Substrate commit.
time cargo test --workspace --release --verbose --features=runtime-benchmarks
#!/bin/sh
#
# check for a polkadot companion pr and ensure it has approvals and is
# mergeable
#
github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls"
# use github api v3 in order to access the data without authentication
github_header="Authorization: token ${GITHUB_PR_TOKEN}"
boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
boldcat <<-EOT
check_polkadot_companion_status
===============================
this job checks if there is a string in the description of the pr like
polkadot companion: paritytech/polkadot#567
and checks its status.
EOT
if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ]
then
boldprint "this doesn't seem to be a pull request"
exit 1
fi
boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \
| sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')"
# get companion if explicitly specified
pr_companion="$(echo "${pr_body}" | sed -n -r \
-e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
-e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
| tail -n 1)"
if [ -z "${pr_companion}" ]
then
boldprint "no companion pr found"
exit 0
fi
boldprint "companion pr: #${pr_companion}"
# check the status of that pull request - needs to be
# approved and mergable
curl -H "${github_header}" -sS -o companion_pr.json \
${github_api_polkadot_pull_url}/${pr_companion}
pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json)
boldprint "Polkadot PR's HEAD SHA: $pr_head_sha"
curl -H "${github_header}" -sS -o companion_pr_reviews.json \
${github_api_polkadot_pull_url}/${pr_companion}/reviews
# If there are any 'CHANGES_REQUESTED' reviews for the *current* review
jq -r -e '.[] | select(.state == "CHANGES_REQUESTED").commit_id' \
< companion_pr_reviews.json > companion_pr_reviews_current.json
while IFS= read -r line; do
if [ "$line" = "$pr_head_sha" ]; then
boldprint "polkadot pr #${pr_companion} has CHANGES_REQUESTED for the latest commit"
exit 1
fi
done < companion_pr_reviews_current.json
# Then we check for at least 1 APPROVED
if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ]; then
boldprint "polkadot pr #${pr_companion} not APPROVED"
exit 1
fi
boldprint "polkadot pr #${pr_companion} state APPROVED"
if jq -e .merged < companion_pr.json >/dev/null
then
boldprint "polkadot pr #${pr_companion} already merged"
exit 0
fi
if jq -e '.mergeable' < companion_pr.json >/dev/null
then
boldprint "polkadot pr #${pr_companion} mergeable"
else
boldprint "polkadot pr #${pr_companion} not mergeable"
exit 1
fi
exit 0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment