From 33324fe01c5b1f341687cef2aa6e767f6acf40f3 Mon Sep 17 00:00:00 2001
From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com>
Date: Wed, 3 Jul 2024 14:01:57 +0300
Subject: [PATCH] approval-voting: Make tests deterministic (#3899)
With random connectivity and latency is hard to actually figure it out a
delta in the benchmarking, so disable them in order to get full
deterministic behaviour when measuring performance.
At least on my machine with this configuration the results for
approval-throughput are really similar between subsequent runs:
```
CPU usage, seconds total per block
approval-distribution 36.9025 3.6902
approval-distribution 36.7579 3.6758
approval-distribution 37.0418 3.7042
approval-distribution 37.0339 3.7034
approval-distribution 36.9342 3.6934
approval-distribution 36.7177 3.6718
approval-voting 52.7756 5.2776
approval-voting 52.5999 5.2600
approval-voting 53.2158 5.3216
approval-voting 53.2493 5.3249
approval-voting 52.8524 5.2852
approval-voting 52.8611 5.2861
approval-voting 52.8210 5.2821
```
---------
Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>
---
.../examples/approvals_no_shows.yaml | 2 ++
.../examples/approvals_throughput.yaml | 2 ++
.../approvals_throughput_best_case.yaml | 3 +++
...als_throughput_no_optimisations_enabled.yaml | 3 +++
.../subsystem-bench/src/lib/approval/mod.rs | 17 +++++++++--------
5 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml
index 146da57d44c..cae1a30914d 100644
--- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml
+++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml
@@ -16,3 +16,5 @@ TestConfiguration:
peer_bandwidth: 524288000000
bandwidth: 524288000000
num_blocks: 10
+ connectivity: 100
+ latency: null
diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml
index 6b17e62c20a..7edb48e302a 100644
--- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml
+++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml
@@ -16,3 +16,5 @@ TestConfiguration:
peer_bandwidth: 524288000000
bandwidth: 524288000000
num_blocks: 10
+ connectivity: 100
+ latency: null
diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml
index e946c28e8ef..7c24f50e6af 100644
--- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml
+++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml
@@ -16,3 +16,6 @@ TestConfiguration:
peer_bandwidth: 524288000000
bandwidth: 524288000000
num_blocks: 10
+ connectivity: 100
+ latency: null
+
diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml
index 8f4b050e72f..fe2402faecc 100644
--- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml
+++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml
@@ -16,3 +16,6 @@ TestConfiguration:
peer_bandwidth: 524288000000
bandwidth: 524288000000
num_blocks: 10
+ connectivity: 100
+ latency: null
+
diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs
index b033be9986f..4ac044ea345 100644
--- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs
+++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs
@@ -698,12 +698,12 @@ impl PeerMessageProducer {
.expect("We can't handle unknown peers")
.clone();
- self.network
- .send_message_from_peer(
- &peer_authority_id,
- protocol_v3::ValidationProtocol::ApprovalDistribution(message.msg).into(),
- )
- .unwrap_or_else(|_| panic!("Network should be up and running {:?}", sent_by));
+ if let Err(err) = self.network.send_message_from_peer(
+ &peer_authority_id,
+ protocol_v3::ValidationProtocol::ApprovalDistribution(message.msg).into(),
+ ) {
+ gum::warn!(target: LOG_TARGET, ?sent_by, ?err, "Validator can not send message");
+ }
}
// Queues a message to be sent by the peer identified by the `sent_by` value.
@@ -994,11 +994,12 @@ pub async fn bench_approvals_run(
"polkadot_parachain_subsystem_bounded_received",
Some(("subsystem_name", "approval-distribution-subsystem")),
|value| {
- gum::info!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric");
+ gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric");
value >= at_least_messages as f64
},
)
.await;
+
gum::info!("Requesting approval votes ms");
for info in &state.blocks {
@@ -1038,7 +1039,7 @@ pub async fn bench_approvals_run(
"polkadot_parachain_subsystem_bounded_received",
Some(("subsystem_name", "approval-distribution-subsystem")),
|value| {
- gum::info!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric");
+ gum::debug!(target: LOG_TARGET, ?value, ?at_least_messages, "Waiting metric");
value >= at_least_messages as f64
},
)
--
GitLab