From d18da3180f7af1b166748b381f12363d7a1d57ac Mon Sep 17 00:00:00 2001
From: Martin Pugh <martin@parity.io>
Date: Wed, 4 Aug 2021 04:27:53 +0200
Subject: [PATCH] [CI] Add manual Gitlab job for running all benchmarks (#3515)
* initial weights job
* add artifact
* revertme: changes to test branch
* add benchmarking instructions
* Revert "revertme: changes to test branch"
This reverts commit 2eab22037223967e66a70a16fda14f58e41c6ced.
* add kusama + westend weights jobs
* fix chevdor comments
* add temporary changes for testing again
* fix
* fix
* test sccache fix
* Revert "add temporary changes for testing again"
This reverts commit bb5a7660151f404994c85abfff31502aac89c1d1.
* whitespace
---
polkadot/.github/ISSUE_TEMPLATE/release.md | 18 ++++++++-
polkadot/.gitlab-ci.yml | 27 ++++++++++++++
polkadot/scripts/run_all_benches.sh | 9 +----
polkadot/scripts/run_benches_for_runtime.sh | 41 +++++++++++++++++++++
4 files changed, 85 insertions(+), 10 deletions(-)
create mode 100755 polkadot/scripts/run_benches_for_runtime.sh
diff --git a/polkadot/.github/ISSUE_TEMPLATE/release.md b/polkadot/.github/ISSUE_TEMPLATE/release.md
index 0f4862e5d51..0516ae4090f 100644
--- a/polkadot/.github/ISSUE_TEMPLATE/release.md
+++ b/polkadot/.github/ISSUE_TEMPLATE/release.md
@@ -134,8 +134,22 @@ date to include them.
### Benchmarks
-Run the benchmarking suite with the new runtime and update any function weights
-if necessary.
+There are three benchmarking machines reserved for updating the weights at
+release-time. To initialise a benchmark run for each production runtime
+(westend, kusama, polkadot):
+* Go to https://gitlab.parity.io/parity/polkadot/-/pipelines?page=1&scope=branches&ref=master
+* Click the link to the last pipeline run for master
+* Start each of the manual jobs:
+ * 'update_westend_weights'
+ * 'update_polkadot_weights'
+ * 'update_kusama_weights'
+* When these jobs have completed (it takes a few hours), a git PATCH file will
+ be available to download as an artifact.
+* On your local machine, branch off master
+* Download the patch file and apply it to your branch with `git patch patchfile.patch`
+* Commit the changes to your branch and submit a PR against master
+* The weights should be (Currently manually) checked to make sure there are no
+ big outliers (i.e., twice or half the weight).
### Polkadot JS
diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml
index f3adf63e825..bb79677c743 100644
--- a/polkadot/.gitlab-ci.yml
+++ b/polkadot/.gitlab-ci.yml
@@ -326,6 +326,33 @@ publish-adder-collator-image:
# this artifact is used in trigger-simnet job
dotenv: ./artifacts/collator.env
+.update_weights: &update-weights
+ stage: build
+ when: manual
+ tags:
+ - weights
+ variables:
+ RUNTIME: polkadot
+ artifacts:
+ paths:
+ - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+ script: |
+ ./scripts/run_benches_for_runtime.sh $RUNTIME
+ git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+
+update_polkadot_weights:
+ <<: *update-weights
+
+update_kusama_weights:
+ <<: *update-weights
+ variables:
+ RUNTIME: kusama
+
+update_westend_weights:
+ <<: *update-weights
+ variables:
+ RUNTIME: westend
+
#### stage: publish
publish-s3-release: &publish-s3
diff --git a/polkadot/scripts/run_all_benches.sh b/polkadot/scripts/run_all_benches.sh
index cfbaab233e8..923013f3515 100755
--- a/polkadot/scripts/run_all_benches.sh
+++ b/polkadot/scripts/run_all_benches.sh
@@ -10,13 +10,6 @@ runtimes=(
westend
)
-# cargo build --locked --release
for runtime in "${runtimes[@]}"; do
- cargo +nightly run --release --features=runtime-benchmarks --locked benchmark --chain "${runtime}-dev" --execution=wasm --wasm-execution=compiled --pallet "*" --extrinsic "*" --repeat 0 | sed -r -e 's/Pallet: "([a-z_:]+)".*/\1/' | uniq | grep -v frame_system > "${runtime}_pallets"
- while read -r line; do
- pallet="$(echo "$line" | cut -d' ' -f1)";
- echo "Runtime: $runtime. Pallet: $pallet";
- cargo +nightly run --release --features=runtime-benchmarks -- benchmark --chain="${runtime}-dev" --steps=50 --repeat=20 --pallet="$pallet" --extrinsic="*" --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
- done < "${runtime}_pallets"
- rm "${runtime}_pallets"
+ "$(dirname "$0")/run_benches_for_runtime.sh" "$runtime"
done
diff --git a/polkadot/scripts/run_benches_for_runtime.sh b/polkadot/scripts/run_benches_for_runtime.sh
new file mode 100755
index 00000000000..874e54a3b04
--- /dev/null
+++ b/polkadot/scripts/run_benches_for_runtime.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Runs all benchmarks for all pallets, for a given runtime, provided by $1
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/substrate/pull/5848
+
+runtime="$1"
+standard_args="--release --locked --features=runtime-benchmarks"
+
+echo "[+] Running all benchmarks for $runtime"
+
+# shellcheck disable=SC2086
+cargo +nightly run $standard_args benchmark \
+ --chain "${runtime}-dev" \
+ --execution=wasm \
+ --wasm-execution=compiled \
+ --pallet "*" \
+ --extrinsic "*" \
+ --repeat 0 | \
+ sed -r -e 's/Pallet: "([a-z_:]+)".*/\1/' | \
+ uniq | \
+ grep -v frame_system > "${runtime}_pallets"
+
+# For each pallet found in the previous command, run benches on each function
+while read -r line; do
+ pallet="$(echo "$line" | cut -d' ' -f1)";
+ echo "Runtime: $runtime. Pallet: $pallet";
+# shellcheck disable=SC2086
+cargo +nightly run $standard_args -- benchmark \
+ --chain="${runtime}-dev" \
+ --steps=50 \
+ --repeat=20 \
+ --pallet="$pallet" \
+ --extrinsic="*" \
+ --execution=wasm \
+ --wasm-execution=compiled \
+ --heap-pages=4096 \
+ --header=./file_header.txt \
+ --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
+done < "${runtime}_pallets"
+rm "${runtime}_pallets"
--
GitLab