From f45293829633ab53f5b60988446316c561fa53f6 Mon Sep 17 00:00:00 2001
From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>
Date: Tue, 12 Apr 2022 13:53:02 +0200
Subject: [PATCH] Add benchmarks to pipeline (#1074)
* [Do not merge] add benchmarks to pipeline
* disable jobs for test
* Revert "[Do not merge] add benchmarks to pipeline"
This reverts commit af3cca09f2bbec7ab75e1e905314356ce4405a6f.
* add weights tag
* add stage
* empty before_script for benches
* run benchmarks on ci6
* disable cargo target dir
* disable pre-cache
* add CARGO_TARGET_DIR var to bench script
* debug
* debug
* fix script path
* divide build and benchmark into 2 jobs
* fix bench build
* small fix
* reduce benchmark output log
* add push github
* add push gh
* add after_script
* typo fix
* typo fix
* fix curl
* fix git config
* add title and body to pr
* final check that everything works
* final check that everything works
* return jobs back
* fix rules
* cancel changes benchmarks.sh
* Apply suggestions from code review
Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>
* rename job
* change benches jobs to run on release-parachains-v*
* remove master refs
Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>
---
cumulus/.gitlab-ci.yml | 97 +++++++++++++++++++++++++-------
cumulus/scripts/benchmarks-ci.sh | 37 ++++++++++++
2 files changed, 115 insertions(+), 19 deletions(-)
create mode 100755 cumulus/scripts/benchmarks-ci.sh
diff --git a/cumulus/.gitlab-ci.yml b/cumulus/.gitlab-ci.yml
index 0d89bd90f2f..0b38b409005 100644
--- a/cumulus/.gitlab-ci.yml
+++ b/cumulus/.gitlab-ci.yml
@@ -8,8 +8,19 @@
stages:
- test
- publish
+ - benchmarks-build
+ - benchmarks-run
-variables: &default-vars
+default:
+ interruptible: true
+ retry:
+ max: 2
+ when:
+ - runner_system_failure
+ - unknown_failure
+ - api_failure
+
+variables:
GIT_STRATEGY: fetch
GIT_DEPTH: 100
CARGO_INCREMENTAL: 0
@@ -17,13 +28,20 @@ variables: &default-vars
DOCKER_OS: "debian:stretch"
ARCH: "x86_64"
+.collect-artifacts: &collect-artifacts
+ artifacts:
+ name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+ when: on_success
+ expire_in: 1 days
+ paths:
+ - ./artifacts/
+
.rust-info-script: &rust-info-script
- rustup show
- cargo --version
- rustup +nightly show
- cargo +nightly --version
- bash --version
- - sccache -s
.publish-refs: &publish-refs
rules:
@@ -31,29 +49,27 @@ variables: &default-vars
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
+# run benchmarks manually only on release-parachains-v* branch
+.benchmarks-manual-refs: &benchmarks-manual-refs
+ rules:
+ - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9]+\.[0-9]+.*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1
+ when: manual
+
+# run benchmarks only on release-parachains-v* branch
+.benchmarks-refs: &benchmarks-refs
+ rules:
+ - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9]+\.[0-9]+.*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1
+ when: manual
+
.docker-env: &docker-env
image: "${CI_IMAGE}"
before_script:
- *rust-info-script
- - sccache -s
- retry:
- max: 2
- when:
- - runner_system_failure
- - unknown_failure
- - api_failure
- interruptible: true
tags:
- linux-docker
.kubernetes-env: &kubernetes-env
- retry:
- max: 2
- when:
- - runner_system_failure
- - unknown_failure
- - api_failure
- interruptible: true
+ image: "${CI_IMAGE}"
tags:
- kubernetes-parity-build
@@ -84,7 +100,6 @@ test-linux-stable:
- if: $CI_PIPELINE_SOURCE == "schedule"
<<: *collect-artifacts
variables:
- <<: *default-vars
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
@@ -136,7 +151,6 @@ check-rustdoc:
stage: test
<<: *docker-env
variables:
- <<: *default-vars
SKIP_WASM_BUILD: 1
RUSTDOCFLAGS: "-Dwarnings"
script:
@@ -169,3 +183,48 @@ publish-s3:
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
--recursive --human-readable --summarize
+
+#### stage: benchmarks
+# Work only on release-parachains-v* branches
+
+benchmarks-build:
+ stage: benchmarks-build
+ <<: *docker-env
+ <<: *collect-artifacts
+ <<: *benchmarks-manual-refs
+ script:
+ - time cargo build --profile production --locked --features runtime-benchmarks
+ - mkdir artifacts
+ - cp $CARGO_TARGET_DIR/production/polkadot-collator ./artifacts/
+
+benchmarks:
+ stage: benchmarks-run
+ before_script:
+ - *rust-info-script
+ <<: *collect-artifacts
+ <<: *benchmarks-refs
+ script:
+ - ./scripts/benchmarks-ci.sh statemine > ./artifacts/bench-statemine.log
+ - ./scripts/benchmarks-ci.sh statemint > ./artifacts/bench-statemint.log
+ - ./scripts/benchmarks-ci.sh westmint > ./artifacts/bench-westmint.log
+ - git status
+ - export BRANCHNAME="${CI_COMMIT_BRANCH}-weights"
+ # Set git config
+ - rm -rf .git/config
+ - git config --global user.email "${GITHUB_EMAIL}"
+ - git config --global user.name "${GITHUB_USER}"
+ - git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com:/paritytech/${CI_PROJECT_NAME}.git"
+ - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+ # push results to github
+ - git checkout -b $BRANCHNAME
+ - git add polkadot-parachains/*
+ - git commit -m "[benchmarks] pr with wieghts"
+ - git push origin $BRANCHNAME
+ # create PR
+ - curl -u ${GITHUB_USER}:${GITHUB_TOKEN}
+ -d '{"title":"[benchmarks] Update weights","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}'
+ -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls
+ after_script:
+ - rm -rf .git/config
+ tags:
+ - weights
diff --git a/cumulus/scripts/benchmarks-ci.sh b/cumulus/scripts/benchmarks-ci.sh
new file mode 100755
index 00000000000..91d24bb5dc7
--- /dev/null
+++ b/cumulus/scripts/benchmarks-ci.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+steps=50
+repeat=20
+chainName=$1
+
+benhcmarkOutput=./polkadot-parachains/$chainName/src/weights
+benhcmarkChainName="$chainName-dev"
+
+pallets=(
+ pallet_assets
+ pallet_balances
+ pallet_collator_selection
+ pallet_multisig
+ pallet_proxy
+ pallet_session
+ pallet_timestamp
+ pallet_utility
+ pallet_uniques
+ cumulus_pallet_xcmp_queue
+ frame_system
+)
+
+for p in ${pallets[@]}
+do
+ ./artifacts/polkadot-collator benchmark \
+ --chain=$benhcmarkChainName \
+ --execution=wasm \
+ --wasm-execution=compiled \
+ --pallet=$p \
+ --extrinsic='*' \
+ --steps=$steps \
+ --repeat=$repeat \
+ --json \
+ --header=./file_header.txt \
+ --output=$benhcmarkOutput
+done
--
GitLab