Use custom runner for import benchmarks (#5477)

* custom benchmark runner

* add license preamble

* delete old benchmarks

* update toml

* imporove macro

* tabs

* add size

* size fixes

* add docs for cli

substrate/Cargo.lock

@@ -3314,6 +3314,21 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "node-bench"
+version = "0.8.0-alpha.5"
+dependencies = [
+ "log",
+ "node-primitives",
+ "node-testing",
+ "sc-cli",
+ "sc-client-api",
+ "serde",
+ "serde_json",
+ "sp-runtime",
+ "structopt",
+]
+
 [[package]]
 name = "node-cli"
 version = "2.0.0-alpha.5"

substrate/Cargo.toml

@@ -3,6 +3,7 @@ members = [
 	"bin/node-template/node",
 	"bin/node-template/runtime",
 	"bin/node-template/pallets/template",
+	"bin/node/bench",
 	"bin/node/cli",
 	"bin/node/executor",
 	"bin/node/primitives",

substrate/bin/node/bench/Cargo.toml

+[package]
+name = "node-bench"
+version = "0.8.0-alpha.5"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Substrate node integration benchmarks."
+edition = "2018"
+license = "GPL-3.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4.8"
+node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" }
+node-testing = { version = "2.0.0-alpha.5", path = "../testing" }
+sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" }
+sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" }
+sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" }
+serde = "1.0.101"
+serde_json = "1.0.41"
+structopt = "0.3"
\ No newline at end of file

substrate/bin/node/bench/src/core.rs

+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt, borrow::{Cow, ToOwned}};
+use serde::Serialize;
+
+pub struct Path(Vec<String>);
+
+impl Path {
+	pub fn new(initial: &'static [&'static str]) -> Self {
+		Path(initial.iter().map(|x| x.to_string()).collect())
+	}
+}
+
+impl Path {
+	pub fn push(&mut self, item: &str) {
+		self.0.push(item.to_string());
+	}
+
+	pub fn full(&self) -> String {
+		self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val })
+	}
+
+	pub fn has(&self, path: &str) -> bool {
+		self.full().contains(path)
+	}
+}
+
+pub trait BenchmarkDescription {
+	fn path(&self) -> Path;
+
+	fn setup(self: Box<Self>) -> Box<dyn Benchmark>;
+
+	fn name(&self) -> Cow<'static, str>;
+}
+
+pub trait Benchmark {
+	fn run(&mut self) -> std::time::Duration;
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct BenchmarkOutput {
+	name: String,
+	raw_average: u64,
+	average: u64,
+}
+
+struct NsFormatter(u64);
+
+impl fmt::Display for NsFormatter {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		let v = self.0;
+
+		if v < 100 {
+			return write!(f, "{} ns", v)
+		}
+
+		if self.0 < 10_000 {
+			return write!(f, "{:.1} µs", v as f64 / 1000.0)
+		}
+
+		if self.0 < 1_000_000 {
+			return write!(f, "{:.1} ms", v as f64 / 1_000_000.0)
+		}
+
+		if self.0 < 100_000_000 {
+			return write!(f, "{} ms", v as f64 / 1_000_000.0)
+		}
+
+		write!(f, "{:.2} s", v as f64 / 1_000_000_000.0)
+	}
+}
+
+impl fmt::Display for BenchmarkOutput {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+			f,
+			"({}: avg {}, w_avg {})",
+			self.name,
+			NsFormatter(self.raw_average),
+			NsFormatter(self.average),
+		)
+    }
+}
+
+pub fn run_benchmark(benchmark: Box<dyn BenchmarkDescription>) -> BenchmarkOutput {
+	let name = benchmark.name().to_owned();
+	let mut benchmark = benchmark.setup();
+
+	let mut durations: Vec<u128> = vec![];
+	for _ in 0..50 {
+		let duration = benchmark.run();
+		durations.push(duration.as_nanos());
+	}
+
+	durations.sort();
+
+	let raw_average = (durations.iter().sum::<u128>() / (durations.len() as u128)) as u64;
+	let average = (durations.iter().skip(10).take(30).sum::<u128>() / 30) as u64;
+
+	BenchmarkOutput {
+		name: name.into(),
+		raw_average,
+		average,
+	}
+}
+
+macro_rules! matrix(
+	( $var:ident in $over:expr => $tt:expr,  $( $rest:tt )* ) => {
+		{
+			let mut res = Vec::<Box<dyn crate::core::BenchmarkDescription>>::new();
+			for $var in $over.iter() {
+				res.push(Box::new($tt));
+			}
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	( $var:expr, $( $rest:tt )*) => {
+		{
+			let mut res = vec![Box::new($var) as Box<dyn crate::core::BenchmarkDescription>];
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	() => { vec![] }
+);
\ No newline at end of file

substrate/bin/node/bench/src/import.rs

+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block import benchmark.
+//!
+//! This benchmark is expected to measure block import operation of
+//! some more or less full block.
+//!
+//! As we also want to protect against cold-cache attacks, this
+//! benchmark should not rely on any caching (except those that
+//! DO NOT depend on user input). Thus block generation should be
+//! based on randomized operation.
+//!
+//! This is supposed to be very simple benchmark and is not subject
+//! to much configuring - just block full of randomized transactions.
+//! It is not supposed to measure runtime modules weight correctness
+
+use std::borrow::Cow;
+
+use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
+use node_primitives::Block;
+use sc_client_api::backend::Backend;
+use sp_runtime::generic::BlockId;
+
+use crate::core::{self, Path};
+
+#[derive(Clone, Copy, Debug)]
+pub enum SizeType { Small, Medium, Large }
+
+impl SizeType {
+	fn transactions(&self) -> usize {
+		match self {
+			SizeType::Small => 10,
+			SizeType::Medium => 100,
+			SizeType::Large => 500,
+		}
+	}
+}
+
+pub struct ImportBenchmarkDescription {
+	pub profile: Profile,
+	pub key_types: KeyTypes,
+	pub size: SizeType,
+}
+
+pub struct ImportBenchmark {
+	profile: Profile,
+	database: BenchDb,
+	block: Block,
+}
+
+impl core::BenchmarkDescription for ImportBenchmarkDescription {
+	fn path(&self) -> Path {
+
+		let mut path = Path::new(&["node", "import"]);
+
+		match self.profile {
+			Profile::Wasm => path.push("wasm"),
+			Profile::Native => path.push("native"),
+		}
+
+		match self.key_types {
+			KeyTypes::Sr25519 => path.push("sr25519"),
+			KeyTypes::Ed25519 => path.push("ed25519"),
+		}
+
+		match self.size {
+			SizeType::Small => path.push("small"),
+			SizeType::Medium => path.push("medium"),
+			SizeType::Large => path.push("large"),
+		}
+
+		path
+	}
+
+	fn setup(self: Box<Self>) -> Box<dyn core::Benchmark> {
+		let profile = self.profile;
+		let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types);
+		let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions()));
+		Box::new(ImportBenchmark {
+			database: bench_db,
+			block,
+			profile,
+		})
+	}
+
+	fn name(&self) -> Cow<'static, str> {
+		match self.profile {
+			Profile::Wasm => "Import benchmark (random transfers, wasm)".into(),
+			Profile::Native => "Import benchmark (random transfers, native)".into(),
+		}
+	}
+}
+
+impl core::Benchmark for ImportBenchmark {
+	fn run(&mut self) -> std::time::Duration {
+		let mut context = self.database.create_context(self.profile);
+
+		let _ = context.client.runtime_version_at(&BlockId::Number(0))
+			.expect("Failed to get runtime version")
+			.spec_version;
+
+		let start = std::time::Instant::now();
+		context.import_block(self.block.clone());
+		let elapsed = start.elapsed();
+
+		log::info!(
+			target: "bench-logistics",
+			"imported block with {} tx, took: {:#?}",
+			self.block.extrinsics.len(),
+			elapsed,
+		);
+
+		log::info!(
+			target: "bench-logistics",
+			"usage info: {}",
+			context.backend.usage_info()
+				.expect("RocksDB backend always provides usage info!"),
+		);
+
+		elapsed
+	}
+}
\ No newline at end of file

substrate/bin/node/bench/src/main.rs

+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#[macro_use] mod core;
+mod import;
+
+use crate::core::run_benchmark;
+use import::{ImportBenchmarkDescription, SizeType};
+use node_testing::bench::{Profile, KeyTypes};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "node-bench", about = "Node integration benchmarks")]
+struct Opt {
+	/// Show list of all available benchmarks.
+	///
+	/// Will output ("name", "path"). Benchmarks can then be filtered by path.
+	#[structopt(short, long)]
+	list: bool,
+
+	/// Machine readable json output.
+	///
+	/// This also suppresses all regular output (except to stderr)
+	#[structopt(short, long)]
+	json: bool,
+
+	/// Filter benchmarks.
+	///
+	/// Run with `--list` for the hint of what to filter.
+	filter: Option<String>,
+}
+
+fn main() {
+	let opt = Opt::from_args();
+
+	if !opt.json {
+		sc_cli::init_logger("");
+	}
+
+	let benchmarks = matrix!(
+		profile in [Profile::Wasm, Profile::Native] =>
+			ImportBenchmarkDescription {
+				profile: *profile,
+				key_types: KeyTypes::Sr25519,
+				size: SizeType::Medium,
+			},
+		ImportBenchmarkDescription {
+			profile: Profile::Native,
+			key_types: KeyTypes::Ed25519,
+			size: SizeType::Medium,
+		},
+		size in [SizeType::Small, SizeType::Large] =>
+			ImportBenchmarkDescription {
+				profile: Profile::Native,
+				key_types: KeyTypes::Sr25519,
+				size: *size,
+			},
+	);
+
+	if opt.list {
+		for benchmark in benchmarks.iter() {
+			log::info!("{}: {}", benchmark.name(), benchmark.path().full())
+		}
+		return;
+	}
+
+	let mut results = Vec::new();
+	for benchmark in benchmarks {
+		if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) {
+			log::info!("Starting {}", benchmark.name());
+			let result = run_benchmark(benchmark);
+			log::info!("{}", result);
+
+			results.push(result);
+		}
+	}
+
+	if opt.json {
+		let json_result: String = serde_json::to_string(&results).expect("Failed to construct json");
+		println!("{}", json_result);
+	}
+}
\ No newline at end of file

substrate/bin/node/testing/Cargo.toml

@@ -53,9 +53,5 @@ criterion = "0.3.0"
 sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" }
 sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service", features = ["rocksdb"] }
 
-[[bench]]
-name = "import"
-harness = false
-
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

substrate/bin/node/testing/benches/import.rs

-// Copyright 2020 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Block import benchmark.
-//!
-//! This benchmark is expected to measure block import operation of
-//! some more or less full block.
-//!
-//! As we also want to protect against cold-cache attacks, this
-//! benchmark should not rely on any caching (except those that
-//! DO NOT depend on user input). Thus block generation should be
-//! based on randomized operation.
-//!
-//! This is supposed to be very simple benchmark and is not subject
-//! to much configuring - just block full of randomized transactions.
-//! It is not supposed to measure runtime modules weight correctness
-
-use std::fmt;
-use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
-use node_primitives::Block;
-use sp_runtime::generic::BlockId;
-use criterion::{Criterion, criterion_group, criterion_main};
-use sc_client_api::backend::Backend;
-
-criterion_group!(
-	name = benches;
-	config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20));
-	targets = bench_block_import, bench_account_reaping, bench_account_ed25519
-);
-criterion_group!(
-	name = wasm_size;
-	config = Criterion::default().sample_size(10);
-	targets = bench_wasm_size_import
-);
-criterion_group!(
-	name = profile;
-	config = Criterion::default().sample_size(10);
-	targets = profile_block_import
-);
-criterion_main!(benches, profile);
-
-fn bench_block_import(c: &mut Criterion) {
-	sc_cli::init_logger("");
-	// for future uses, uncomment if something wrong.
-	// sc_cli::init_logger("sc_client=debug");
-
-	let mut bench_db = BenchDb::new(100);
-	let block = bench_db.generate_block(BlockType::RandomTransfers(100));
-
-	log::trace!(
-		target: "bench-logistics",
-		"Seed database directory: {}",
-		bench_db.path().display(),
-	);
-
-	c.bench_function_over_inputs("import-block-B-0001",
-		move |bencher, profile| {
-			bencher.iter_batched(
-				|| {
-					let context = bench_db.create_context(*profile);
-
-					// mostly to just launch compiler before benching!
-					let version = context.client.runtime_version_at(&BlockId::Number(0))
-						.expect("Failed to get runtime version")
-						.spec_version;
-
-					log::trace!(
-						target: "bench-logistics",
-						"Next iteration database directory: {}, runtime version: {}",
-						context.path().display(), version,
-					);
-
-					context
-				},
-				|mut context| {
-					let start = std::time::Instant::now();
-					context.import_block(block.clone());
-					let elapsed = start.elapsed();
-
-					log::info!(
-						target: "bench-logistics",
-						"imported block with {} tx, took: {:#?}",
-						block.extrinsics.len(),
-						elapsed,
-					);
-
-					log::info!(
-						target: "bench-logistics",
-						"usage info: {}",
-						context.backend.usage_info()
-							.expect("RocksDB backend always provides usage info!"),
-					);
-				},
-				criterion::BatchSize::LargeInput,
-			);
-		},
-		vec![Profile::Wasm, Profile::Native],
-	);
-}
-
-fn bench_account_reaping(c: &mut Criterion) {
-	sc_cli::init_logger("");
-
-	let mut bench_db = BenchDb::new(100);
-	let block = bench_db.generate_block(BlockType::RandomTransfersReaping(100));
-
-	c.bench_function_over_inputs("import-block-reaping-B-0002",
-		move |bencher, profile| {
-			bencher.iter_batched(
-				|| {
-					let context = bench_db.create_context(*profile);
-
-					// mostly to just launch compiler before benching!
-					context.client.runtime_version_at(&BlockId::Number(0))
-						.expect("Failed to get runtime version");
-
-					context
-				},
-				|mut context| {
-					context.import_block(block.clone());
-				},
-				criterion::BatchSize::LargeInput,
-			);
-		},
-		vec![Profile::Wasm, Profile::Native],
-	);
-}
-
-fn bench_account_ed25519(c: &mut Criterion) {
-	sc_cli::init_logger("");
-
-	let mut bench_db = BenchDb::with_key_types(100, KeyTypes::Ed25519);
-	let block = bench_db.generate_block(BlockType::RandomTransfers(100));
-
-	c.bench_function_over_inputs("import-block-ed25519-B-0003",
-		move |bencher, profile| {
-			bencher.iter_batched(
-				|| {
-					let context = bench_db.create_context(*profile);
-					context.client.runtime_version_at(&BlockId::Number(0))
-						.expect("Failed to get runtime version");
-
-					context
-				},
-				|mut context| {
-					context.import_block(block.clone());
-				},
-				criterion::BatchSize::LargeInput,
-			);
-		},
-		vec![Profile::Wasm, Profile::Native],
-	);
-}
-
-// This is not an actual benchmark, so don't use it to measure anything.
-//   It just produces special pattern of cpu load that allows easy picking
-//   the part of block import for the profiling in the tool of choice.
-fn profile_block_import(c: &mut Criterion) {
-	sc_cli::init_logger("");
-
-	let mut bench_db = BenchDb::new(128);
-	let block = bench_db.generate_block(BlockType::RandomTransfers(100));
-
-	c.bench_function("profile block",
-		move |bencher| {
-			bencher.iter_batched(
-				|| {
-					bench_db.create_context(Profile::Native)
-				},
-				|mut context| {
-					// until better osx signpost/callgrind signal is possible to use
-					// in rust, we just pause everything completely to help choosing
-					// actual profiling interval
-					std::thread::park_timeout(std::time::Duration::from_secs(2));
-					context.import_block(block.clone());
-					// and here as well
-					std::thread::park_timeout(std::time::Duration::from_secs(2));
-					log::info!(
-						target: "bench-logistics",
-						"imported block, usage info: {}",
-						context.backend.usage_info()
-							.expect("RocksDB backend always provides usage info!"),
-					)
-				},
-				criterion::BatchSize::PerIteration,
-			);
-		},
-	);
-}
-
-struct Setup {
-	db: BenchDb,
-	block: Block,
-}
-
-struct SetupIterator {
-	current: usize,
-	finish: usize,
-	multiplier: usize,
-}
-
-impl SetupIterator {
-	fn new(current: usize, finish: usize, multiplier: usize) -> Self {
-		SetupIterator { current, finish, multiplier }
-	}
-}
-
-impl Iterator for SetupIterator {
-	type Item = Setup;
-
-	fn next(&mut self) -> Option<Setup> {
-		if self.current >= self.finish { return None }
-
-		self.current += 1;
-
-		let size = self.current * self.multiplier;
-		let mut db = BenchDb::new(size);
-		let block = db.generate_block(BlockType::RandomTransfers(size));
-		Some(Setup { db, block })
-	}
-}
-
-impl fmt::Debug for Setup {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Setup: {} tx/block", self.block.extrinsics.len())
-    }
-}
-
-fn bench_wasm_size_import(c: &mut Criterion) {
-	sc_cli::init_logger("");
-
-	c.bench_function_over_inputs("wasm_size_import",
-		move |bencher, setup| {
-			bencher.iter_batched(
-				|| {
-					setup.db.create_context(Profile::Wasm)
-				},
-				|mut context| {
-					context.import_block(setup.block.clone());
-				},
-				criterion::BatchSize::PerIteration,
-			);
-		},
-		SetupIterator::new(5, 15, 50),
-	);
-}