Run all benchmarks "as tests" in short setup on every PR (#2846)

* Run all benchmarks "as tests" in short setup on every PR

* Fix

* Update scripts/ci/gitlab/pipeline/short-benchmarks.yml

Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>

* Updated TODO

* Removed `--execution wasm`

* [asset-conv] Unused Balances type

* [asset-conv] native asset change

* Dedicated `AssetBalance` type for `pallet_assets` instances

* Improved local vs foreign asset handling + test for not allowing pool_assets in pool

* Removed `into_multiasset_id`

* Fix

* Refactor

* Fixed create_pool for benchmark with LocalAndForeignAssets (finally found it)

* Revert

* fmt

* Migrates pools with `MultiLocation { parents: 0, interior: Here }` to `MultiLocation { parents: 1, interior: Here }`

* Allow `set_storage` for `AllowMultiAssetPools` / `LiquidityWithdrawalFee`

* Benchmarks work

* Removed comment + more defensive migration

* `T::Currency::transfer` -> `Balances::transfer_all` in migration

*...

cumulus/.gitlab-ci.yml

@@ -7,11 +7,14 @@
 stages:
   - test
   - build
+  # used for manual job run for regenerate weights for release-* branches (not needed anymore, just leave it here for a while as PlanB)
   - benchmarks-build
+  # used for manual job run for regenerate weights for release-* branches (not needed anymore, just leave it here for a while as PlanB)
   - benchmarks-run
   - publish
   - integration-tests
   - zombienet
+  - short-benchmarks
 
 default:
   interruptible: true
@@ -142,7 +145,10 @@ include:
   - scripts/ci/gitlab/pipeline/test.yml
   # # build jobs
   - scripts/ci/gitlab/pipeline/build.yml
+  # short-benchmarks jobs
+  - scripts/ci/gitlab/pipeline/short-benchmarks.yml
   # # benchmarks jobs
+  # # used for manual job run for regenerate weights for release-* branches (not needed anymore, just leave it here for a while as PlanB)
   - scripts/ci/gitlab/pipeline/benchmarks.yml
   # # publish jobs
   - scripts/ci/gitlab/pipeline/publish.yml

cumulus/scripts/ci/gitlab/pipeline/build.yml

@@ -121,3 +121,18 @@ build-runtime-testing:
   needs:
     - job: build-runtime-starters
       artifacts: false
+
+build-short-benchmark:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .collect-artifacts
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job: check-rustdoc
+      artifacts: false
+  script:
+    - cargo build --profile release --locked --features=runtime-benchmarks
+    - mkdir -p ./artifacts
+    - cp ./target/release/polkadot-parachain ./artifacts/

cumulus/scripts/ci/gitlab/pipeline/short-benchmarks.yml

+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "short-benchmarks" stage
+
+# Run all pallet benchmarks only once to check if there are any errors
+.short-benchmark-template: &short-bench
+  stage: short-benchmarks
+  extends:
+    - .common-refs
+    - .docker-env
+  needs:
+    - job: build-short-benchmark
+      artifacts: true
+  variables:
+    RUNTIME_CHAIN: benchmarked-runtime-chain
+  script:
+    - ./artifacts/polkadot-parachain benchmark pallet --wasm-execution compiled --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+
+short-benchmark-asset-hub-polkadot:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: asset-hub-polkadot-dev
+
+short-benchmark-asset-hub-kusama:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: asset-hub-kusama-dev
+
+short-benchmark-asset-hub-westend:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: asset-hub-westend-dev
+
+short-benchmark-bridge-hub-polkadot:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: bridge-hub-polkadot-dev
+
+short-benchmark-bridge-hub-kusama:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: bridge-hub-kusama-dev
+
+short-benchmark-bridge-hub-rococo:
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: bridge-hub-rococo-dev
+
+short-benchmark-collectives-polkadot :
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: collectives-polkadot-dev
+
+short-benchmark-glutton-kusama :
+  <<: *short-bench
+  variables:
+    RUNTIME_CHAIN: glutton-kusama-dev-1300