From 8af9889f561fcf97ac7c1446ceb72f9324348dd4 Mon Sep 17 00:00:00 2001
From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com>
Date: Wed, 7 Aug 2024 17:22:27 +0300
Subject: [PATCH] Bring reference_hardware.json inline with machine used for
weights (#5196)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Since `May 2023` after
https://github.com/paritytech/substrate/pull/13548 optimization,
`Blake2256` is faster with about 30%, that means that there is a
difference of ~30% between the benchmark values we ask validators to run
against and the machine we use for generating the weights.So if all
validators, just barely pass the benchmarks our weights are potentially
underestimated with about ~20%, so let's bring this two in sync.
Same thing happened when we merged
https://github.com/paritytech/polkadot-sdk/pull/2524 in `Nov 2023`
SR25519-Verify became faster with about 10-15%
## Results
Generated on machine from here:
https://github.com/paritytech/devops/pull/3210
```
+----------+----------------+--------------+-------------+-------------------+
| Category | Function | Score | Minimum | Result |
+============================================================================+
| CPU | BLAKE2-256 | 1.00 GiBs | 783.27 MiBs | ✅ Pass (130.7 %) |
|----------+----------------+--------------+-------------+-------------------|
| CPU | SR25519-Verify | 637.62 KiBs | 560.67 KiBs | ✅ Pass (113.7 %) |
|----------+----------------+--------------+-------------+-------------------|
| Memory | Copy | 12.19 GiBs | 11.49 GiBs | ✅ Pass (106.1 %) |
```
Discovered and discussed here:
https://github.com/paritytech/polkadot-sdk/pull/5127#issuecomment-2258423469
## Downsides
Machines that barely passed the benchmark will suddenly find themselves
bellow the benchmark, but since that is just an warning and everything
else continues as before it shouldn't be too impactful and should give
the validators the necessary information that they need to become
compliant, since they actually aren't when compared with the used
weights.
---------
Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>
---
prdoc/pr_5196.prdoc | 23 +++++++++++++++++++
.../benchmarking-cli/src/machine/hardware.rs | 4 ++--
.../src/machine/reference_hardware.json | 4 ++--
3 files changed, 27 insertions(+), 4 deletions(-)
create mode 100644 prdoc/pr_5196.prdoc
diff --git a/prdoc/pr_5196.prdoc b/prdoc/pr_5196.prdoc
new file mode 100644
index 00000000000..3ed4fbdff3f
--- /dev/null
+++ b/prdoc/pr_5196.prdoc
@@ -0,0 +1,23 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Bring benchmark inline with reference machine used for weights
+
+doc:
+ - audience: Node Operator
+ description: |
+ - BLAKE2-256 reference values were too low(~30%) when compared with the machine used for generating
+ the weights, so it was brought in sync with results on the reference hardware recommended here:
+ https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware
+ - SR25519-Verify reference values were too low(~10%) when compared with the machine used for generating
+ the weights, so it was brought in sync with results on the reference hardware recommended here:
+ https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware
+ - Validators where the `BLAKE2-256` and `SR25519-Verify` were barely passing, might received the
+ warning that they are not compliant anymore, this should not be treated as critical, but they
+ should take the necessary steps to become compliant in the near/mid-term future.
+ - Note!: The reference hardware requirements have not been increased we just fixed the benchmark which
+ was wrongly reporting lower spec HW as being compliant.
+
+crates:
+ - name: frame-benchmarking-cli
+ bump: minor
diff --git a/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs b/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs
index 5a4b7c797b6..555e848f8cc 100644
--- a/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs
+++ b/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs
@@ -51,10 +51,10 @@ mod tests {
assert_eq!(
*SUBSTRATE_REFERENCE_HARDWARE,
Requirements(vec![
- Requirement { metric: Metric::Blake2256, minimum: Throughput::from_mibs(783.27) },
+ Requirement { metric: Metric::Blake2256, minimum: Throughput::from_mibs(1000.00) },
Requirement {
metric: Metric::Sr25519Verify,
- minimum: Throughput::from_kibs(560.670000128),
+ minimum: Throughput::from_kibs(637.619999744),
},
Requirement {
metric: Metric::MemCopy,
diff --git a/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json b/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json
index c2fb4c7d4a2..cec42b8f245 100644
--- a/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json
+++ b/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json
@@ -1,11 +1,11 @@
[
{
"metric": "Blake2256",
- "minimum": 783.27
+ "minimum": 1000.00
},
{
"metric": "Sr25519Verify",
- "minimum": 0.547529297
+ "minimum": 0.622675781
},
{
"metric": "MemCopy",
--
GitLab