From 1c695b815e72d3b118ee32221d4ce1c21155decd Mon Sep 17 00:00:00 2001
From: ordian <write@reusable.software>
Date: Mon, 16 Jan 2023 12:04:54 +0100
Subject: [PATCH] add erasure-coding benches (#6308)
* add erasure-code benches
* revert Cargo.lock changes
* revert Cargo.lock changes
---
polkadot/Cargo.lock | 135 ++++++++++++++++--
polkadot/erasure-coding/Cargo.toml | 7 +
polkadot/erasure-coding/benches/README.md | 39 +++++
.../benches/scaling_with_validators.rs | 74 ++++++++++
4 files changed, 244 insertions(+), 11 deletions(-)
create mode 100644 polkadot/erasure-coding/benches/README.md
create mode 100644 polkadot/erasure-coding/benches/scaling_with_validators.rs
diff --git a/polkadot/Cargo.lock b/polkadot/Cargo.lock
index 90701f6a756..1896ef6ca44 100644
--- a/polkadot/Cargo.lock
+++ b/polkadot/Cargo.lock
@@ -152,6 +152,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbf688625d06217d5b1bb0ea9d9c44a1635fd0ee3534466388d18203174f4d11"
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
[[package]]
name = "ansi_term"
version = "0.12.1"
@@ -756,6 +762,12 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
[[package]]
name = "cc"
version = "1.0.73"
@@ -844,6 +856,33 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "ciborium"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
[[package]]
name = "cid"
version = "0.8.6"
@@ -895,6 +934,18 @@ dependencies = [
"libloading",
]
+[[package]]
+name = "clap"
+version = "3.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+dependencies = [
+ "bitflags",
+ "clap_lex 0.2.4",
+ "indexmap",
+ "textwrap",
+]
+
[[package]]
name = "clap"
version = "4.0.15"
@@ -904,7 +955,7 @@ dependencies = [
"atty",
"bitflags",
"clap_derive",
- "clap_lex",
+ "clap_lex 0.3.0",
"once_cell",
"strsim",
"termcolor",
@@ -923,6 +974,15 @@ dependencies = [
"syn",
]
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
[[package]]
name = "clap_lex"
version = "0.3.0"
@@ -1186,6 +1246,40 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap 3.2.23",
+ "criterion-plot",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
[[package]]
name = "crossbeam-channel"
version = "0.5.5"
@@ -2226,7 +2320,7 @@ dependencies = [
"Inflector",
"array-bytes",
"chrono",
- "clap",
+ "clap 4.0.15",
"comfy-table",
"frame-benchmarking",
"frame-support",
@@ -2817,6 +2911,12 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
[[package]]
name = "handlebars"
version = "4.2.2"
@@ -4808,6 +4908,12 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
[[package]]
name = "opaque-debug"
version = "0.2.3"
@@ -6313,7 +6419,7 @@ dependencies = [
name = "polkadot-cli"
version = "0.9.33"
dependencies = [
- "clap",
+ "clap 4.0.15",
"frame-benchmarking-cli",
"futures",
"log",
@@ -6456,6 +6562,7 @@ dependencies = [
name = "polkadot-erasure-coding"
version = "0.9.33"
dependencies = [
+ "criterion",
"parity-scale-codec",
"polkadot-node-primitives",
"polkadot-primitives",
@@ -7570,7 +7677,7 @@ version = "0.9.33"
dependencies = [
"assert_matches",
"async-trait",
- "clap",
+ "clap 4.0.15",
"color-eyre",
"futures",
"futures-timer",
@@ -7716,7 +7823,7 @@ dependencies = [
name = "polkadot-voter-bags"
version = "0.9.33"
dependencies = [
- "clap",
+ "clap 4.0.15",
"generate-bags",
"kusama-runtime",
"polkadot-runtime",
@@ -8343,7 +8450,7 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
name = "remote-ext-tests-bags-list"
version = "0.9.33"
dependencies = [
- "clap",
+ "clap 4.0.15",
"frame-system",
"kusama-runtime",
"kusama-runtime-constants",
@@ -8844,7 +8951,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#934d42aefb51
dependencies = [
"array-bytes",
"chrono",
- "clap",
+ "clap 4.0.15",
"fdlimit",
"futures",
"libp2p",
@@ -10972,7 +11079,7 @@ name = "staking-miner"
version = "0.9.33"
dependencies = [
"assert_cmd",
- "clap",
+ "clap 4.0.15",
"exitcode",
"frame-election-provider-support",
"frame-remote-externalities",
@@ -11432,7 +11539,7 @@ dependencies = [
name = "test-parachain-adder-collator"
version = "0.9.33"
dependencies = [
- "clap",
+ "clap 4.0.15",
"futures",
"futures-timer",
"log",
@@ -11479,7 +11586,7 @@ dependencies = [
name = "test-parachain-undying-collator"
version = "0.9.33"
dependencies = [
- "clap",
+ "clap 4.0.15",
"futures",
"futures-timer",
"log",
@@ -11525,6 +11632,12 @@ dependencies = [
"sp-weights",
]
+[[package]]
+name = "textwrap"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+
[[package]]
name = "thiserror"
version = "1.0.37"
@@ -12066,7 +12179,7 @@ name = "try-runtime-cli"
version = "0.10.0-dev"
source = "git+https://github.com/paritytech/substrate?branch=master#934d42aefb51b797ee9ef41270bc041b1c1c6025"
dependencies = [
- "clap",
+ "clap 4.0.15",
"frame-remote-externalities",
"frame-try-runtime",
"hex",
diff --git a/polkadot/erasure-coding/Cargo.toml b/polkadot/erasure-coding/Cargo.toml
index 6d59b6a4b01..868d1197efd 100644
--- a/polkadot/erasure-coding/Cargo.toml
+++ b/polkadot/erasure-coding/Cargo.toml
@@ -12,3 +12,10 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
thiserror = "1.0.31"
+
+[dev-dependencies]
+criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support"] }
+
+[[bench]]
+name = "scaling_with_validators"
+harness = false
diff --git a/polkadot/erasure-coding/benches/README.md b/polkadot/erasure-coding/benches/README.md
new file mode 100644
index 00000000000..e643643229e
--- /dev/null
+++ b/polkadot/erasure-coding/benches/README.md
@@ -0,0 +1,39 @@
+### Run benches
+```
+$ cd erasure-coding # ensure you are in the right directory
+$ cargo bench
+```
+
+### `scaling_with_validators`
+
+This benchmark evaluates the performance of constructing the chunks and the erasure root from PoV and
+reconstructing the PoV from chunks. You can see the results of running this bench on 5950x below.
+Interestingly, with `10_000` chunks (validators) its slower than with `50_000` for both construction
+and reconstruction.
+```
+construct/200 time: [93.924 ms 94.525 ms 95.214 ms]
+ thrpt: [52.513 MiB/s 52.896 MiB/s 53.234 MiB/s]
+construct/500 time: [111.25 ms 111.52 ms 111.80 ms]
+ thrpt: [44.721 MiB/s 44.837 MiB/s 44.946 MiB/s]
+construct/1000 time: [117.37 ms 118.28 ms 119.21 ms]
+ thrpt: [41.941 MiB/s 42.273 MiB/s 42.601 MiB/s]
+construct/2000 time: [125.05 ms 125.72 ms 126.38 ms]
+ thrpt: [39.564 MiB/s 39.772 MiB/s 39.983 MiB/s]
+construct/10000 time: [270.46 ms 275.11 ms 279.81 ms]
+ thrpt: [17.869 MiB/s 18.174 MiB/s 18.487 MiB/s]
+construct/50000 time: [205.86 ms 209.66 ms 213.64 ms]
+ thrpt: [23.404 MiB/s 23.848 MiB/s 24.288 MiB/s]
+
+reconstruct/200 time: [180.73 ms 184.09 ms 187.73 ms]
+ thrpt: [26.634 MiB/s 27.160 MiB/s 27.666 MiB/s]
+reconstruct/500 time: [195.59 ms 198.58 ms 201.76 ms]
+ thrpt: [24.781 MiB/s 25.179 MiB/s 25.564 MiB/s]
+reconstruct/1000 time: [207.92 ms 211.57 ms 215.57 ms]
+ thrpt: [23.195 MiB/s 23.633 MiB/s 24.048 MiB/s]
+reconstruct/2000 time: [218.59 ms 223.68 ms 229.18 ms]
+ thrpt: [21.817 MiB/s 22.354 MiB/s 22.874 MiB/s]
+reconstruct/10000 time: [496.35 ms 505.17 ms 515.42 ms]
+ thrpt: [9.7008 MiB/s 9.8977 MiB/s 10.074 MiB/s]
+reconstruct/50000 time: [276.56 ms 277.53 ms 278.58 ms]
+ thrpt: [17.948 MiB/s 18.016 MiB/s 18.079 MiB/s]
+```
diff --git a/polkadot/erasure-coding/benches/scaling_with_validators.rs b/polkadot/erasure-coding/benches/scaling_with_validators.rs
new file mode 100644
index 00000000000..d361c23537c
--- /dev/null
+++ b/polkadot/erasure-coding/benches/scaling_with_validators.rs
@@ -0,0 +1,74 @@
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use polkadot_primitives::v2::Hash;
+use std::time::Duration;
+
+fn chunks(n_validators: usize, pov: &Vec<u8>) -> Vec<Vec<u8>> {
+ polkadot_erasure_coding::obtain_chunks(n_validators, pov).unwrap()
+}
+
+fn erasure_root(n_validators: usize, pov: &Vec<u8>) -> Hash {
+ let chunks = chunks(n_validators, pov);
+ polkadot_erasure_coding::branches(&chunks).root()
+}
+
+fn construct_and_reconstruct_5mb_pov(c: &mut Criterion) {
+ const N_VALIDATORS: [usize; 6] = [200, 500, 1000, 2000, 10_000, 50_000];
+
+ const KB: usize = 1024;
+ const MB: usize = 1024 * KB;
+
+ let pov = vec![0xfe; 5 * MB];
+
+ let mut group = c.benchmark_group("construct");
+ for n_validators in N_VALIDATORS {
+ let expected_root = erasure_root(n_validators, &pov);
+
+ group.throughput(Throughput::Bytes(pov.len() as u64));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(n_validators),
+ &n_validators,
+ |b, &n| {
+ b.iter(|| {
+ let root = erasure_root(n, &pov);
+ assert_eq!(root, expected_root);
+ });
+ },
+ );
+ }
+ group.finish();
+
+ let mut group = c.benchmark_group("reconstruct");
+ for n_validators in N_VALIDATORS {
+ let all_chunks = chunks(n_validators, &pov);
+
+ let mut c: Vec<_> = all_chunks.iter().enumerate().map(|(i, c)| (&c[..], i)).collect();
+ let last_chunks = c.split_off((c.len() - 1) * 2 / 3);
+
+ group.throughput(Throughput::Bytes(pov.len() as u64));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(n_validators),
+ &n_validators,
+ |b, &n| {
+ b.iter(|| {
+ let _pov: Vec<u8> =
+ polkadot_erasure_coding::reconstruct(n, last_chunks.clone()).unwrap();
+ });
+ },
+ );
+ }
+ group.finish();
+}
+
+fn criterion_config() -> Criterion {
+ Criterion::default()
+ .sample_size(15)
+ .warm_up_time(Duration::from_millis(200))
+ .measurement_time(Duration::from_secs(3))
+}
+
+criterion_group!(
+ name = re_construct;
+ config = criterion_config();
+ targets = construct_and_reconstruct_5mb_pov,
+);
+criterion_main!(re_construct);
--
GitLab