use pipeline-scripts for dependent projects script (#9749)

substrate/.gitlab-ci.yml

@@ -46,6 +46,7 @@ variables:                         &default-vars
   VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
   VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
   SIMNET_FEATURES_PATH:            "simnet_tests/tests"
+  PIPELINE_SCRIPTS_TAG:            "v0.1"
 
 default:
   cache:                           {}
@@ -538,29 +539,37 @@ cargo-check-macos:
 
 #### stage:                        build
 
-check-polkadot-companion-status:
-  stage:                           build
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-env
-  <<:                              *vault-secrets
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-  script:
-    - ./.maintain/gitlab/check_polkadot_companion_status.sh
-
-check-polkadot-companion-build:
+.check-dependent-project:          &check-dependent-project
   stage:                           build
   <<:                              *docker-env
   <<:                              *test-refs-no-trigger
   <<:                              *vault-secrets
-  needs:
-    - job:                         test-linux-stable-int
-      artifacts:                   false
   script:
-    - ./.maintain/gitlab/check_polkadot_companion_build.sh
-  after_script:
-    - cd polkadot && git rev-parse --abbrev-ref HEAD
-  allow_failure:                   true
+    - git clone
+        --depth=1
+        "--branch=$PIPELINE_SCRIPTS_TAG"
+        https://github.com/paritytech/pipeline-scripts
+    - ./pipeline-scripts/check_dependent_project.sh
+        paritytech
+        substrate
+        --substrate
+        "$DEPENDENT_REPO"
+        "$GITHUB_PR_TOKEN"
+    - cd "$DEPENDENT_REPO" && git rev-parse --abbrev-ref HEAD
+
+# Individual jobs are set up for each dependent project so that they can be ran in parallel.
+# Arguably we could generate a job for each companion in the PR's description using Gitlab's
+# parent-child pipelines but that's more complicated.
+
+check-dependent-polkadot:
+  <<: *check-dependent-project
+  variables:
+    DEPENDENT_REPO: polkadot
+
+check-dependent-cumulus:
+  <<: *check-dependent-project
+  variables:
+    DEPENDENT_REPO: cumulus
 
 build-linux-substrate:             &build-binary
   stage:                           build
@@ -884,7 +893,7 @@ deploy-prometheus-alerting-rules:
 # Runs "quick" and "long" tests on nightly schedule and on commit / merge to master
 # A "quick" test is a smoke test where basic check-expect tests run by
 # checking values from metrics exposed by the app.
-# A "long" test is the load testing where we send 50K transactions into the 
+# A "long" test is the load testing where we send 50K transactions into the
 # network and check if all completed successfully
 simnet-tests:
   stage:                           deploy

substrate/.maintain/gitlab/check_polkadot_companion_build.sh

-#!/usr/bin/env sh
-#
-# check if a pr is compatible with polkadot companion pr or master if not
-# available
-#
-# to override one that was just mentioned mark companion pr in the body of the
-# polkadot pr like
-#
-# polkadot companion: paritytech/polkadot#567
-#
-
-set -e
-
-github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
-# use github api v3 in order to access the data without authentication
-github_header="Authorization: token ${GITHUB_PR_TOKEN}"
-
-boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
-boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
-
-
-
-boldcat <<-EOT
-
-
-check_polkadot_companion_build
-==============================
-
-this job checks if there is a string in the description of the pr like
-
-polkadot companion: paritytech/polkadot#567
-
-
-it will then run cargo check from this polkadot's branch with substrate code
-from this pull request. otherwise, it will uses master instead
-
-
-EOT
-
-# Set the user name and email to make merging work
-git config --global user.name 'CI system'
-git config --global user.email '<>'
-
-# Merge master into our branch before building Polkadot to make sure we don't miss
-# any commits that are required by Polkadot.
-git fetch --depth 100 origin
-git merge origin/master
-
-# Clone the current Polkadot master branch into ./polkadot.
-# NOTE: we need to pull enough commits to be able to find a common
-# ancestor for successfully performing merges below.
-git clone --depth 20 https://github.com/paritytech/polkadot.git
-
-cd polkadot
-
-# either it's a pull request then check for a companion otherwise use
-# polkadot:master
-if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null
-then
-  boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
-
-  pr_data_file="$(mktemp)"
-  # get the last reference to a pr in polkadot
-  curl -sSL -H "${github_header}" -o "${pr_data_file}" \
-    "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}"
-
-  pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")"
-
-  pr_companion="$(echo "${pr_body}" | sed -n -r \
-      -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
-      -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
-    | tail -n 1)"
-
-  if [ "${pr_companion}" ]
-  then
-    boldprint "companion pr specified/detected: #${pr_companion}"
-    git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion}
-    git checkout pr/${pr_companion}
-    git merge origin/master
-  else
-    boldprint "no companion branch found - building polkadot:master"
-  fi
-  rm -f "${pr_data_file}"
-else
-  boldprint "this is not a pull request - building polkadot:master"
-fi
-
-# Patch all Substrate crates in Polkadot
-diener patch --crates-to-patch ../ --substrate --path Cargo.toml
-
-# We need to update specifically our patched Substrate crates so that other
-# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version
-# NOTE: There's no way to only update patched crates, so we use a heuristic
-# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of
-# updating unrelated dependencies
-cargo update -p sp-core
-
-# Test Polkadot pr or master branch with this Substrate commit.
-time cargo test --workspace --release --verbose --features=runtime-benchmarks

substrate/.maintain/gitlab/check_polkadot_companion_status.sh

-#!/bin/sh
-#
-# check for a polkadot companion pr and ensure it has approvals and is
-# mergeable
-#
-
-github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
-github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls"
-# use github api v3 in order to access the data without authentication
-github_header="Authorization: token ${GITHUB_PR_TOKEN}"
-
-boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
-boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
-
-
-
-boldcat <<-EOT
-
-
-check_polkadot_companion_status
-===============================
-
-this job checks if there is a string in the description of the pr like
-
-polkadot companion: paritytech/polkadot#567
-
-and checks its status.
-
-
-EOT
-
-
-if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ]
-then
-  boldprint "this doesn't seem to be a pull request"
-  exit 1
-fi
-
-boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
-
-pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \
-  | sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')"
-
-# get companion if explicitly specified
-pr_companion="$(echo "${pr_body}" | sed -n -r \
-    -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
-    -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
-  | tail -n 1)"
-
-if [ -z "${pr_companion}" ]
-then
-  boldprint "no companion pr found"
-  exit 0
-fi
-
-boldprint "companion pr: #${pr_companion}"
-
-# check the status of that pull request - needs to be
-# approved and mergable
-
-curl -H "${github_header}" -sS -o companion_pr.json \
-  ${github_api_polkadot_pull_url}/${pr_companion}
-
-pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json)
-boldprint "Polkadot PR's HEAD SHA: $pr_head_sha"
-
-curl -H "${github_header}" -sS -o companion_pr_reviews.json \
-  ${github_api_polkadot_pull_url}/${pr_companion}/reviews
-
-# If there are any 'CHANGES_REQUESTED' reviews for the *current* review
-jq -r -e '.[] | select(.state == "CHANGES_REQUESTED").commit_id' \
-  < companion_pr_reviews.json > companion_pr_reviews_current.json
-while IFS= read -r line; do
-  if [ "$line" = "$pr_head_sha" ]; then
-    boldprint "polkadot pr #${pr_companion} has CHANGES_REQUESTED for the latest commit"
-    exit 1
-  fi
-done < companion_pr_reviews_current.json
-
-# Then we check for at least 1 APPROVED
-if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ]; then
-  boldprint "polkadot pr #${pr_companion} not APPROVED"
-  exit 1
-fi
-
-boldprint "polkadot pr #${pr_companion} state APPROVED"
-
-if jq -e .merged < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} already merged"
-  exit 0
-fi
-
-if jq -e '.mergeable' < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} mergeable"
-else
-  boldprint "polkadot pr #${pr_companion} not mergeable"
-  exit 1
-fi
-
-exit 0