From cedb1d43a0b6654d2de3f5213548aacebd41ce80 Mon Sep 17 00:00:00 2001
From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>
Date: Tue, 14 May 2024 16:01:47 +0200
Subject: [PATCH] [ci] Cleanup and order GHA workflows (#4449)
Github currently [doesn't support creating
subfolders](https://github.com/orgs/community/discussions/18055) in
`.github/workflows` folder. PR unifies naming in this folder. The idea
is that all files ideally should have naming
`<stage>-<short_descriptive_name>.yml`. The stage names are taken from
gitlab and needed only for logical division.
Changes in actions:
- `check-licences` runs on self-hosted runners
- `check-workspace` and `check-markdown` are moved to `checks-quick.yml`
- `test-linux-stable-int` moved to a new file `tests-linux-stable.yml`
- `fmt-check.yml` file is removed, formatting is checked in
`checks-quick.yml`
---
.github/workflows/check-licenses.yml | 3 +-
.github/workflows/check-markdown.yml | 35 ----------
.github/workflows/check-runtime-migration.yml | 3 +
.github/workflows/check-workspace.yml | 21 ------
.../{quick-checks.yml => checks-quick.yml} | 48 ++++++++++++-
.github/workflows/fmt-check.yml | 23 -------
...sues.yml => issues-auto-add-parachain.yml} | 0
...label-issues.yml => issues-auto-label.yml} | 0
...label.yml => misc-notify-burnin-label.yml} | 0
...ue.yml => misc-review-bot-merge-queue.yml} | 7 +-
...-templates.yml => misc-sync-templates.yml} | 0
...k-publish.yml => publish-check-crates.yml} | 0
...im-crates.yml => publish-claim-crates.yml} | 0
...s.yml => publish-subsystem-benchmarks.yml} | 0
... => release-build-and-attach-runtimes.yml} | 0
...untimes.yml => release-check-runtimes.yml} | 0
.../{srtool.yml => release-srtool.yml} | 0
.github/workflows/tests-linux-stable.yml | 69 +++++++++++++++++++
.github/workflows/tests.yml | 21 +-----
.gitlab/pipeline/test.yml | 2 -
20 files changed, 126 insertions(+), 106 deletions(-)
delete mode 100644 .github/workflows/check-markdown.yml
delete mode 100644 .github/workflows/check-workspace.yml
rename .github/workflows/{quick-checks.yml => checks-quick.yml} (63%)
delete mode 100644 .github/workflows/fmt-check.yml
rename .github/workflows/{auto-add-parachain-issues.yml => issues-auto-add-parachain.yml} (100%)
rename .github/workflows/{auto-label-issues.yml => issues-auto-label.yml} (100%)
rename .github/workflows/{notif-burnin-label.yml => misc-notify-burnin-label.yml} (100%)
rename .github/workflows/{merge-queue.yml => misc-review-bot-merge-queue.yml} (79%)
rename .github/workflows/{sync-templates.yml => misc-sync-templates.yml} (100%)
rename .github/workflows/{check-publish.yml => publish-check-crates.yml} (100%)
rename .github/workflows/{claim-crates.yml => publish-claim-crates.yml} (100%)
rename .github/workflows/{subsystem-benchmarks.yml => publish-subsystem-benchmarks.yml} (100%)
rename .github/workflows/{build-and-attach-release-runtimes.yml => release-build-and-attach-runtimes.yml} (100%)
rename .github/workflows/{check-runtimes.yml => release-check-runtimes.yml} (100%)
rename .github/workflows/{srtool.yml => release-srtool.yml} (100%)
create mode 100644 .github/workflows/tests-linux-stable.yml
diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml
index c32b6fcf89e..31716b1a04a 100644
--- a/.github/workflows/check-licenses.yml
+++ b/.github/workflows/check-licenses.yml
@@ -9,7 +9,8 @@ permissions:
jobs:
check-licenses:
- runs-on: ubuntu-latest
+ runs-on: arc-runners-polkadot-sdk
+ timeout-minutes: 10
env:
LICENSES: "'Apache-2.0' 'GPL-3.0-only' 'GPL-3.0-or-later WITH Classpath-exception-2.0'"
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/check-markdown.yml b/.github/workflows/check-markdown.yml
deleted file mode 100644
index 2b8a66db35b..00000000000
--- a/.github/workflows/check-markdown.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Check Markdown
-
-on:
- pull_request:
- types: [opened, synchronize, reopened, ready_for_review]
- merge_group:
-
-permissions:
- packages: read
-
-jobs:
- lint-markdown:
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout sources
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
- - uses: actions/setup-node@v4.0.1
- with:
- node-version: "18.x"
- registry-url: "https://npm.pkg.github.com"
- scope: "@paritytech"
-
- - name: Install tooling
- run: |
- npm install -g markdownlint-cli
- markdownlint --version
-
- - name: Check Markdown
- env:
- CONFIG: .github/.markdownlint.yaml
- run: |
- echo "Checking markdown formatting. More info: docs/contributor/markdown_linting.md"
- markdownlint --config "$CONFIG" --ignore target .
diff --git a/.github/workflows/check-runtime-migration.yml b/.github/workflows/check-runtime-migration.yml
index 5de576e2fe4..984e264d0d1 100644
--- a/.github/workflows/check-runtime-migration.yml
+++ b/.github/workflows/check-runtime-migration.yml
@@ -1,6 +1,9 @@
name: check-runtime-migration
on:
+ push:
+ branches:
+ - master
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
merge_group:
diff --git a/.github/workflows/check-workspace.yml b/.github/workflows/check-workspace.yml
deleted file mode 100644
index 81ec311ccce..00000000000
--- a/.github/workflows/check-workspace.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-name: Check workspace
-
-on:
- pull_request:
- merge_group:
-
-jobs:
- check-workspace:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
-
- - name: install python deps
- run: pip3 install toml
-
- - name: check integrity
- run: >
- python3 .github/scripts/check-workspace.py .
- --exclude
- "substrate/frame/contracts/fixtures/build"
- "substrate/frame/contracts/fixtures/contracts/common"
diff --git a/.github/workflows/quick-checks.yml b/.github/workflows/checks-quick.yml
similarity index 63%
rename from .github/workflows/quick-checks.yml
rename to .github/workflows/checks-quick.yml
index a56bf12bb16..1fbf83e3465 100644
--- a/.github/workflows/quick-checks.yml
+++ b/.github/workflows/checks-quick.yml
@@ -2,6 +2,9 @@
name: quick-checks
on:
+ push:
+ branches:
+ - master
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
merge_group:
@@ -9,6 +12,8 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
+permissions: {}
+
jobs:
set-image:
# GitHub Actions allows using 'env' in a container context.
@@ -36,9 +41,9 @@ jobs:
check-dependency-rules:
runs-on: arc-runners-polkadot-sdk
timeout-minutes: 10
- needs: [set-image]
- container:
- image: ${{ needs.set-image.outputs.IMAGE }}
+ # needs: [set-image]
+ # container:
+ # image: ${{ needs.set-image.outputs.IMAGE }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: check dependency rules
@@ -77,3 +82,40 @@ jobs:
run: |
taplo format --check --config .config/taplo.toml
echo "Please run `taplo format --config .config/taplo.toml` to fix any toml formatting issues"
+ check-workspace:
+ runs-on: arc-runners-polkadot-sdk
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+ - name: install python deps
+ run: |
+ sudo apt-get update && sudo apt-get install -y python3-pip python3
+ pip3 install toml
+ - name: check integrity
+ run: >
+ python3 .github/scripts/check-workspace.py .
+ --exclude
+ "substrate/frame/contracts/fixtures/build"
+ "substrate/frame/contracts/fixtures/contracts/common"
+ check-markdown:
+ runs-on: arc-runners-polkadot-sdk
+ timeout-minutes: 10
+ steps:
+ - name: Checkout sources
+ uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+ - name: Setup Node.js
+ uses: actions/setup-node@v4.0.1
+ with:
+ node-version: "18.x"
+ registry-url: "https://npm.pkg.github.com"
+ scope: "@paritytech"
+ - name: Install tooling
+ run: |
+ npm install -g markdownlint-cli
+ markdownlint --version
+ - name: Check Markdown
+ env:
+ CONFIG: .github/.markdownlint.yaml
+ run: |
+ echo "Checking markdown formatting. More info: docs/contributor/markdown_linting.md"
+ markdownlint --config "$CONFIG" --ignore target .
diff --git a/.github/workflows/fmt-check.yml b/.github/workflows/fmt-check.yml
deleted file mode 100644
index 324c9bfff7a..00000000000
--- a/.github/workflows/fmt-check.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Rustfmt check
-
-on:
- push:
- branches:
- - master
- pull_request:
- types: [opened, synchronize, reopened, ready_for_review]
- merge_group:
-
-jobs:
- quick_check:
- strategy:
- matrix:
- os: ["ubuntu-latest"]
- runs-on: ${{ matrix.os }}
- container:
- image: docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408
- steps:
- - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
- - name: Cargo fmt
- run: cargo +nightly fmt --all -- --check
diff --git a/.github/workflows/auto-add-parachain-issues.yml b/.github/workflows/issues-auto-add-parachain.yml
similarity index 100%
rename from .github/workflows/auto-add-parachain-issues.yml
rename to .github/workflows/issues-auto-add-parachain.yml
diff --git a/.github/workflows/auto-label-issues.yml b/.github/workflows/issues-auto-label.yml
similarity index 100%
rename from .github/workflows/auto-label-issues.yml
rename to .github/workflows/issues-auto-label.yml
diff --git a/.github/workflows/notif-burnin-label.yml b/.github/workflows/misc-notify-burnin-label.yml
similarity index 100%
rename from .github/workflows/notif-burnin-label.yml
rename to .github/workflows/misc-notify-burnin-label.yml
diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/misc-review-bot-merge-queue.yml
similarity index 79%
rename from .github/workflows/merge-queue.yml
rename to .github/workflows/misc-review-bot-merge-queue.yml
index cce326f4493..28b3bc7533c 100644
--- a/.github/workflows/merge-queue.yml
+++ b/.github/workflows/misc-review-bot-merge-queue.yml
@@ -1,3 +1,4 @@
+# Actions that makes review-bot green in the merge queue
name: Merge-Queue
on:
@@ -18,7 +19,7 @@ jobs:
uses: billyjbryant/create-status-check@3e6fa0ac599d10d9588cf9516ca4330ef669b858 # v2
with:
authToken: ${{ steps.app_token.outputs.token }}
- context: 'review-bot'
- description: 'PRs for merge queue gets approved'
- state: 'success'
+ context: "review-bot"
+ description: "PRs for merge queue gets approved"
+ state: "success"
sha: ${{ github.event.merge_group.head_commit.id }}
diff --git a/.github/workflows/sync-templates.yml b/.github/workflows/misc-sync-templates.yml
similarity index 100%
rename from .github/workflows/sync-templates.yml
rename to .github/workflows/misc-sync-templates.yml
diff --git a/.github/workflows/check-publish.yml b/.github/workflows/publish-check-crates.yml
similarity index 100%
rename from .github/workflows/check-publish.yml
rename to .github/workflows/publish-check-crates.yml
diff --git a/.github/workflows/claim-crates.yml b/.github/workflows/publish-claim-crates.yml
similarity index 100%
rename from .github/workflows/claim-crates.yml
rename to .github/workflows/publish-claim-crates.yml
diff --git a/.github/workflows/subsystem-benchmarks.yml b/.github/workflows/publish-subsystem-benchmarks.yml
similarity index 100%
rename from .github/workflows/subsystem-benchmarks.yml
rename to .github/workflows/publish-subsystem-benchmarks.yml
diff --git a/.github/workflows/build-and-attach-release-runtimes.yml b/.github/workflows/release-build-and-attach-runtimes.yml
similarity index 100%
rename from .github/workflows/build-and-attach-release-runtimes.yml
rename to .github/workflows/release-build-and-attach-runtimes.yml
diff --git a/.github/workflows/check-runtimes.yml b/.github/workflows/release-check-runtimes.yml
similarity index 100%
rename from .github/workflows/check-runtimes.yml
rename to .github/workflows/release-check-runtimes.yml
diff --git a/.github/workflows/srtool.yml b/.github/workflows/release-srtool.yml
similarity index 100%
rename from .github/workflows/srtool.yml
rename to .github/workflows/release-srtool.yml
diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml
new file mode 100644
index 00000000000..8822ba6d250
--- /dev/null
+++ b/.github/workflows/tests-linux-stable.yml
@@ -0,0 +1,69 @@
+# GHA for test-linux-stable-int, test-linux-stable, test-linux-stable-oldkernel
+name: tests linux stable
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ types: [opened, synchronize, reopened, ready_for_review]
+ merge_group:
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+env:
+ FORKLIFT_storage_s3_bucketName: ${{ secrets.FORKLIFT_storage_s3_bucketName }}
+ FORKLIFT_storage_s3_accessKeyId: ${{ secrets.FORKLIFT_storage_s3_accessKeyId }}
+ FORKLIFT_storage_s3_secretAccessKey: ${{ secrets.FORKLIFT_storage_s3_secretAccessKey }}
+ FORKLIFT_storage_s3_endpointUrl: ${{ secrets.FORKLIFT_storage_s3_endpointUrl }}
+ FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
+
+jobs:
+ set-image:
+ # GitHub Actions allows using 'env' in a container context.
+ # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+ # This workaround sets the container image for each job using 'set-image' job output.
+ runs-on: ubuntu-latest
+ outputs:
+ IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - id: set_image
+ run: cat .github/env >> $GITHUB_OUTPUT
+ test-linux-stable-int:
+ runs-on: arc-runners-polkadot-sdk-beefy
+ timeout-minutes: 30
+ needs: [set-image]
+ container:
+ image: ${{ needs.set-image.outputs.IMAGE }}
+ env:
+ RUSTFLAGS: "-C debug-assertions -D warnings"
+ RUST_BACKTRACE: 1
+ WASM_BUILD_NO_COLOR: 1
+ WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
+ # Ensure we run the UI tests.
+ RUN_UI_TESTS: 1
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: script
+ run: WASM_BUILD_NO_COLOR=1 time forklift cargo test -p staging-node-cli --release --locked -- --ignored
+ # https://github.com/paritytech/ci_cd/issues/864
+ test-linux-stable-runtime-benchmarks:
+ runs-on: arc-runners-polkadot-sdk-beefy
+ timeout-minutes: 30
+ needs: [set-image]
+ container:
+ image: ${{ needs.set-image.outputs.IMAGE }}
+ env:
+ RUST_TOOLCHAIN: stable
+ # Enable debug assertions since we are running optimized builds for testing
+ # but still want to have debug assertions.
+ RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: script
+ run: time forklift cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8f46959cccc..76bccba86b2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,6 +1,9 @@
name: tests
on:
+ push:
+ branches:
+ - master
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
merge_group:
@@ -28,24 +31,6 @@ jobs:
uses: actions/checkout@v4
- id: set_image
run: cat .github/env >> $GITHUB_OUTPUT
- test-linux-stable-int:
- runs-on: arc-runners-polkadot-sdk-beefy
- timeout-minutes: 30
- needs: [set-image]
- container:
- image: ${{ needs.set-image.outputs.IMAGE }}
- env:
- RUSTFLAGS: "-C debug-assertions -D warnings"
- RUST_BACKTRACE: 1
- WASM_BUILD_NO_COLOR: 1
- WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
- # Ensure we run the UI tests.
- RUN_UI_TESTS: 1
- steps:
- - name: Checkout
- uses: actions/checkout@v4
- - name: script
- run: WASM_BUILD_NO_COLOR=1 time forklift cargo test -p staging-node-cli --release --locked -- --ignored
quick-benchmarks:
runs-on: arc-runners-polkadot-sdk-beefy
timeout-minutes: 30
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
index 7976c2ad2e0..eea561ff8ba 100644
--- a/.gitlab/pipeline/test.yml
+++ b/.gitlab/pipeline/test.yml
@@ -128,8 +128,6 @@ test-linux-stable:
script:
# Build all but only execute 'runtime' tests.
- echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
- # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged
- # "upgrade_version_checks_should_work" is currently failing
- >
time cargo nextest run \
--workspace \
--
GitLab