Skip to content
Snippets Groups Projects
Commit 6eea144f authored by Nikolay Volf's avatar Nikolay Volf Committed by GitHub
Browse files

Use custom runner for import benchmarks (#5477)

* custom benchmark runner

* add license preamble

* delete old benchmarks

* update toml

* imporove macro

* tabs

* add size

* size fixes

* add docs for cli
parent 68169039
Branches
No related merge requests found
......@@ -3314,6 +3314,21 @@ dependencies = [
"void",
]
[[package]]
name = "node-bench"
version = "0.8.0-alpha.5"
dependencies = [
"log",
"node-primitives",
"node-testing",
"sc-cli",
"sc-client-api",
"serde",
"serde_json",
"sp-runtime",
"structopt",
]
[[package]]
name = "node-cli"
version = "2.0.0-alpha.5"
......
......@@ -3,6 +3,7 @@ members = [
"bin/node-template/node",
"bin/node-template/runtime",
"bin/node-template/pallets/template",
"bin/node/bench",
"bin/node/cli",
"bin/node/executor",
"bin/node/primitives",
......
[package]
name = "node-bench"
version = "0.8.0-alpha.5"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Substrate node integration benchmarks."
edition = "2018"
license = "GPL-3.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
log = "0.4.8"
node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" }
node-testing = { version = "2.0.0-alpha.5", path = "../testing" }
sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" }
sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" }
sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" }
serde = "1.0.101"
serde_json = "1.0.41"
structopt = "0.3"
\ No newline at end of file
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use std::{fmt, borrow::{Cow, ToOwned}};
use serde::Serialize;
pub struct Path(Vec<String>);
impl Path {
pub fn new(initial: &'static [&'static str]) -> Self {
Path(initial.iter().map(|x| x.to_string()).collect())
}
}
impl Path {
pub fn push(&mut self, item: &str) {
self.0.push(item.to_string());
}
pub fn full(&self) -> String {
self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val })
}
pub fn has(&self, path: &str) -> bool {
self.full().contains(path)
}
}
pub trait BenchmarkDescription {
fn path(&self) -> Path;
fn setup(self: Box<Self>) -> Box<dyn Benchmark>;
fn name(&self) -> Cow<'static, str>;
}
pub trait Benchmark {
fn run(&mut self) -> std::time::Duration;
}
#[derive(Debug, Clone, Serialize)]
pub struct BenchmarkOutput {
name: String,
raw_average: u64,
average: u64,
}
struct NsFormatter(u64);
impl fmt::Display for NsFormatter {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let v = self.0;
if v < 100 {
return write!(f, "{} ns", v)
}
if self.0 < 10_000 {
return write!(f, "{:.1} µs", v as f64 / 1000.0)
}
if self.0 < 1_000_000 {
return write!(f, "{:.1} ms", v as f64 / 1_000_000.0)
}
if self.0 < 100_000_000 {
return write!(f, "{} ms", v as f64 / 1_000_000.0)
}
write!(f, "{:.2} s", v as f64 / 1_000_000_000.0)
}
}
impl fmt::Display for BenchmarkOutput {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"({}: avg {}, w_avg {})",
self.name,
NsFormatter(self.raw_average),
NsFormatter(self.average),
)
}
}
pub fn run_benchmark(benchmark: Box<dyn BenchmarkDescription>) -> BenchmarkOutput {
let name = benchmark.name().to_owned();
let mut benchmark = benchmark.setup();
let mut durations: Vec<u128> = vec![];
for _ in 0..50 {
let duration = benchmark.run();
durations.push(duration.as_nanos());
}
durations.sort();
let raw_average = (durations.iter().sum::<u128>() / (durations.len() as u128)) as u64;
let average = (durations.iter().skip(10).take(30).sum::<u128>() / 30) as u64;
BenchmarkOutput {
name: name.into(),
raw_average,
average,
}
}
macro_rules! matrix(
( $var:ident in $over:expr => $tt:expr, $( $rest:tt )* ) => {
{
let mut res = Vec::<Box<dyn crate::core::BenchmarkDescription>>::new();
for $var in $over.iter() {
res.push(Box::new($tt));
}
res.extend(matrix!( $($rest)* ));
res
}
};
( $var:expr, $( $rest:tt )*) => {
{
let mut res = vec![Box::new($var) as Box<dyn crate::core::BenchmarkDescription>];
res.extend(matrix!( $($rest)* ));
res
}
};
() => { vec![] }
);
\ No newline at end of file
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Block import benchmark.
//!
//! This benchmark is expected to measure block import operation of
//! some more or less full block.
//!
//! As we also want to protect against cold-cache attacks, this
//! benchmark should not rely on any caching (except those that
//! DO NOT depend on user input). Thus block generation should be
//! based on randomized operation.
//!
//! This is supposed to be very simple benchmark and is not subject
//! to much configuring - just block full of randomized transactions.
//! It is not supposed to measure runtime modules weight correctness
use std::borrow::Cow;
use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
use node_primitives::Block;
use sc_client_api::backend::Backend;
use sp_runtime::generic::BlockId;
use crate::core::{self, Path};
#[derive(Clone, Copy, Debug)]
pub enum SizeType { Small, Medium, Large }
impl SizeType {
fn transactions(&self) -> usize {
match self {
SizeType::Small => 10,
SizeType::Medium => 100,
SizeType::Large => 500,
}
}
}
pub struct ImportBenchmarkDescription {
pub profile: Profile,
pub key_types: KeyTypes,
pub size: SizeType,
}
pub struct ImportBenchmark {
profile: Profile,
database: BenchDb,
block: Block,
}
impl core::BenchmarkDescription for ImportBenchmarkDescription {
fn path(&self) -> Path {
let mut path = Path::new(&["node", "import"]);
match self.profile {
Profile::Wasm => path.push("wasm"),
Profile::Native => path.push("native"),
}
match self.key_types {
KeyTypes::Sr25519 => path.push("sr25519"),
KeyTypes::Ed25519 => path.push("ed25519"),
}
match self.size {
SizeType::Small => path.push("small"),
SizeType::Medium => path.push("medium"),
SizeType::Large => path.push("large"),
}
path
}
fn setup(self: Box<Self>) -> Box<dyn core::Benchmark> {
let profile = self.profile;
let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types);
let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions()));
Box::new(ImportBenchmark {
database: bench_db,
block,
profile,
})
}
fn name(&self) -> Cow<'static, str> {
match self.profile {
Profile::Wasm => "Import benchmark (random transfers, wasm)".into(),
Profile::Native => "Import benchmark (random transfers, native)".into(),
}
}
}
impl core::Benchmark for ImportBenchmark {
fn run(&mut self) -> std::time::Duration {
let mut context = self.database.create_context(self.profile);
let _ = context.client.runtime_version_at(&BlockId::Number(0))
.expect("Failed to get runtime version")
.spec_version;
let start = std::time::Instant::now();
context.import_block(self.block.clone());
let elapsed = start.elapsed();
log::info!(
target: "bench-logistics",
"imported block with {} tx, took: {:#?}",
self.block.extrinsics.len(),
elapsed,
);
log::info!(
target: "bench-logistics",
"usage info: {}",
context.backend.usage_info()
.expect("RocksDB backend always provides usage info!"),
);
elapsed
}
}
\ No newline at end of file
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
#[macro_use] mod core;
mod import;
use crate::core::run_benchmark;
use import::{ImportBenchmarkDescription, SizeType};
use node_testing::bench::{Profile, KeyTypes};
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
#[structopt(name = "node-bench", about = "Node integration benchmarks")]
struct Opt {
/// Show list of all available benchmarks.
///
/// Will output ("name", "path"). Benchmarks can then be filtered by path.
#[structopt(short, long)]
list: bool,
/// Machine readable json output.
///
/// This also suppresses all regular output (except to stderr)
#[structopt(short, long)]
json: bool,
/// Filter benchmarks.
///
/// Run with `--list` for the hint of what to filter.
filter: Option<String>,
}
fn main() {
let opt = Opt::from_args();
if !opt.json {
sc_cli::init_logger("");
}
let benchmarks = matrix!(
profile in [Profile::Wasm, Profile::Native] =>
ImportBenchmarkDescription {
profile: *profile,
key_types: KeyTypes::Sr25519,
size: SizeType::Medium,
},
ImportBenchmarkDescription {
profile: Profile::Native,
key_types: KeyTypes::Ed25519,
size: SizeType::Medium,
},
size in [SizeType::Small, SizeType::Large] =>
ImportBenchmarkDescription {
profile: Profile::Native,
key_types: KeyTypes::Sr25519,
size: *size,
},
);
if opt.list {
for benchmark in benchmarks.iter() {
log::info!("{}: {}", benchmark.name(), benchmark.path().full())
}
return;
}
let mut results = Vec::new();
for benchmark in benchmarks {
if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) {
log::info!("Starting {}", benchmark.name());
let result = run_benchmark(benchmark);
log::info!("{}", result);
results.push(result);
}
}
if opt.json {
let json_result: String = serde_json::to_string(&results).expect("Failed to construct json");
println!("{}", json_result);
}
}
\ No newline at end of file
......@@ -53,9 +53,5 @@ criterion = "0.3.0"
sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" }
sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service", features = ["rocksdb"] }
[[bench]]
name = "import"
harness = false
[package.metadata.docs.rs]
targets = ["x86_64-unknown-linux-gnu"]
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Block import benchmark.
//!
//! This benchmark is expected to measure block import operation of
//! some more or less full block.
//!
//! As we also want to protect against cold-cache attacks, this
//! benchmark should not rely on any caching (except those that
//! DO NOT depend on user input). Thus block generation should be
//! based on randomized operation.
//!
//! This is supposed to be very simple benchmark and is not subject
//! to much configuring - just block full of randomized transactions.
//! It is not supposed to measure runtime modules weight correctness
use std::fmt;
use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
use node_primitives::Block;
use sp_runtime::generic::BlockId;
use criterion::{Criterion, criterion_group, criterion_main};
use sc_client_api::backend::Backend;
criterion_group!(
name = benches;
config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20));
targets = bench_block_import, bench_account_reaping, bench_account_ed25519
);
criterion_group!(
name = wasm_size;
config = Criterion::default().sample_size(10);
targets = bench_wasm_size_import
);
criterion_group!(
name = profile;
config = Criterion::default().sample_size(10);
targets = profile_block_import
);
criterion_main!(benches, profile);
fn bench_block_import(c: &mut Criterion) {
sc_cli::init_logger("");
// for future uses, uncomment if something wrong.
// sc_cli::init_logger("sc_client=debug");
let mut bench_db = BenchDb::new(100);
let block = bench_db.generate_block(BlockType::RandomTransfers(100));
log::trace!(
target: "bench-logistics",
"Seed database directory: {}",
bench_db.path().display(),
);
c.bench_function_over_inputs("import-block-B-0001",
move |bencher, profile| {
bencher.iter_batched(
|| {
let context = bench_db.create_context(*profile);
// mostly to just launch compiler before benching!
let version = context.client.runtime_version_at(&BlockId::Number(0))
.expect("Failed to get runtime version")
.spec_version;
log::trace!(
target: "bench-logistics",
"Next iteration database directory: {}, runtime version: {}",
context.path().display(), version,
);
context
},
|mut context| {
let start = std::time::Instant::now();
context.import_block(block.clone());
let elapsed = start.elapsed();
log::info!(
target: "bench-logistics",
"imported block with {} tx, took: {:#?}",
block.extrinsics.len(),
elapsed,
);
log::info!(
target: "bench-logistics",
"usage info: {}",
context.backend.usage_info()
.expect("RocksDB backend always provides usage info!"),
);
},
criterion::BatchSize::LargeInput,
);
},
vec![Profile::Wasm, Profile::Native],
);
}
fn bench_account_reaping(c: &mut Criterion) {
sc_cli::init_logger("");
let mut bench_db = BenchDb::new(100);
let block = bench_db.generate_block(BlockType::RandomTransfersReaping(100));
c.bench_function_over_inputs("import-block-reaping-B-0002",
move |bencher, profile| {
bencher.iter_batched(
|| {
let context = bench_db.create_context(*profile);
// mostly to just launch compiler before benching!
context.client.runtime_version_at(&BlockId::Number(0))
.expect("Failed to get runtime version");
context
},
|mut context| {
context.import_block(block.clone());
},
criterion::BatchSize::LargeInput,
);
},
vec![Profile::Wasm, Profile::Native],
);
}
fn bench_account_ed25519(c: &mut Criterion) {
sc_cli::init_logger("");
let mut bench_db = BenchDb::with_key_types(100, KeyTypes::Ed25519);
let block = bench_db.generate_block(BlockType::RandomTransfers(100));
c.bench_function_over_inputs("import-block-ed25519-B-0003",
move |bencher, profile| {
bencher.iter_batched(
|| {
let context = bench_db.create_context(*profile);
context.client.runtime_version_at(&BlockId::Number(0))
.expect("Failed to get runtime version");
context
},
|mut context| {
context.import_block(block.clone());
},
criterion::BatchSize::LargeInput,
);
},
vec![Profile::Wasm, Profile::Native],
);
}
// This is not an actual benchmark, so don't use it to measure anything.
// It just produces special pattern of cpu load that allows easy picking
// the part of block import for the profiling in the tool of choice.
fn profile_block_import(c: &mut Criterion) {
sc_cli::init_logger("");
let mut bench_db = BenchDb::new(128);
let block = bench_db.generate_block(BlockType::RandomTransfers(100));
c.bench_function("profile block",
move |bencher| {
bencher.iter_batched(
|| {
bench_db.create_context(Profile::Native)
},
|mut context| {
// until better osx signpost/callgrind signal is possible to use
// in rust, we just pause everything completely to help choosing
// actual profiling interval
std::thread::park_timeout(std::time::Duration::from_secs(2));
context.import_block(block.clone());
// and here as well
std::thread::park_timeout(std::time::Duration::from_secs(2));
log::info!(
target: "bench-logistics",
"imported block, usage info: {}",
context.backend.usage_info()
.expect("RocksDB backend always provides usage info!"),
)
},
criterion::BatchSize::PerIteration,
);
},
);
}
struct Setup {
db: BenchDb,
block: Block,
}
struct SetupIterator {
current: usize,
finish: usize,
multiplier: usize,
}
impl SetupIterator {
fn new(current: usize, finish: usize, multiplier: usize) -> Self {
SetupIterator { current, finish, multiplier }
}
}
impl Iterator for SetupIterator {
type Item = Setup;
fn next(&mut self) -> Option<Setup> {
if self.current >= self.finish { return None }
self.current += 1;
let size = self.current * self.multiplier;
let mut db = BenchDb::new(size);
let block = db.generate_block(BlockType::RandomTransfers(size));
Some(Setup { db, block })
}
}
impl fmt::Debug for Setup {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Setup: {} tx/block", self.block.extrinsics.len())
}
}
fn bench_wasm_size_import(c: &mut Criterion) {
sc_cli::init_logger("");
c.bench_function_over_inputs("wasm_size_import",
move |bencher, setup| {
bencher.iter_batched(
|| {
setup.db.create_context(Profile::Wasm)
},
|mut context| {
context.import_block(setup.block.clone());
},
criterion::BatchSize::PerIteration,
);
},
SetupIterator::new(5, 15, 50),
);
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment