> {
- match key.map(|k| H256::from_str(&k)) {
- Some(Ok(secret)) => Ok(Some(secret.into())),
- Some(Err(err)) => Err(create_input_err(format!("Error parsing node key: {}", err))),
- None => Ok(None),
+/// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context
+/// of an optional network config storage directory.
+fn node_key_config(params: NodeKeyParams, net_config_dir: &Option
)
+ -> error::Result
+where
+ P: AsRef
+{
+ match params.node_key_type {
+ NodeKeyType::Secp256k1 =>
+ params.node_key.as_ref().map(parse_secp256k1_secret).unwrap_or_else(||
+ Ok(params.node_key_file
+ .or_else(|| net_config_file(net_config_dir, NODE_KEY_SECP256K1_FILE))
+ .map(network::Secret::File)
+ .unwrap_or(network::Secret::New)))
+ .map(NodeKeyConfig::Secp256k1),
+
+ NodeKeyType::Ed25519 =>
+ params.node_key.as_ref().map(parse_ed25519_secret).unwrap_or_else(||
+ Ok(params.node_key_file
+ .or_else(|| net_config_file(net_config_dir, NODE_KEY_ED25519_FILE))
+ .map(network::Secret::File)
+ .unwrap_or(network::Secret::New)))
+ .map(NodeKeyConfig::Ed25519)
}
}
+fn net_config_file(net_config_dir: &Option
, name: &str) -> Option
+where
+ P: AsRef
+{
+ net_config_dir.as_ref().map(|d| d.as_ref().join(name))
+}
+
+/// Create an error caused by an invalid node key argument.
+fn invalid_node_key(e: impl std::fmt::Display) -> error::Error {
+ input_err(format!("Invalid node key: {}", e))
+}
+
+/// Parse a Secp256k1 secret key from a hex string into a `network::Secret`.
+fn parse_secp256k1_secret(hex: &String) -> error::Result {
+ H256::from_str(hex).map_err(invalid_node_key).and_then(|bytes|
+ network::identity::secp256k1::SecretKey::from_bytes(bytes)
+ .map(network::Secret::Input)
+ .map_err(invalid_node_key))
+}
+
+/// Parse a Ed25519 secret key from a hex string into a `network::Secret`.
+fn parse_ed25519_secret(hex: &String) -> error::Result {
+ H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes|
+ network::identity::ed25519::SecretKey::from_bytes(bytes)
+ .map(network::Secret::Input)
+ .map_err(invalid_node_key))
+}
+
/// Fill the given `PoolConfiguration` by looking at the cli parameters.
fn fill_transaction_pool_configuration(
options: &mut FactoryFullConfiguration,
@@ -295,11 +354,13 @@ fn fill_network_configuration(
config.public_addresses = Vec::new();
config.client_version = client_id;
- config.use_secret = parse_node_key(cli.node_key)?;
+ config.node_key = node_key_config(cli.node_key_params, &config.net_config_path)?;
config.in_peers = cli.in_peers;
config.out_peers = cli.out_peers;
+ config.enable_mdns = !cli.no_mdns;
+
Ok(())
}
@@ -317,14 +378,14 @@ where
config.impl_commit = version.commit;
config.impl_version = version.version;
- config.name = match cli.name {
+ config.name = match cli.name.or(cli.keyring.account.map(|a| a.to_string())) {
None => generate_node_name(),
Some(name) => name,
};
match is_node_name_valid(&config.name) {
Ok(_) => (),
Err(msg) => bail!(
- create_input_err(
+ input_err(
format!("Invalid node name '{}'. Reason: {}. If unsure, use none.",
config.name,
msg
@@ -347,7 +408,7 @@ where
Some(ref s) if s == "archive" => PruningMode::ArchiveAll,
None => PruningMode::default(),
Some(s) => PruningMode::keep_blocks(
- s.parse().map_err(|_| create_input_err("Invalid pruning mode specified"))?
+ s.parse().map_err(|_| input_err("Invalid pruning mode specified"))?
),
};
@@ -360,14 +421,25 @@ where
service::Roles::FULL
};
+ let exec = cli.execution_strategies;
config.execution_strategies = ExecutionStrategies {
- syncing: cli.syncing_execution.into(),
- importing: cli.importing_execution.into(),
- block_construction: cli.block_construction_execution.into(),
- other: cli.other_execution.into(),
+ syncing: exec.syncing_execution.into(),
+ importing: exec.importing_execution.into(),
+ block_construction: exec.block_construction_execution.into(),
+ offchain_worker: exec.offchain_worker_execution.into(),
+ other: exec.other_execution.into(),
+ };
+
+ config.offchain_worker = match (cli.offchain_worker, role) {
+ (params::OffchainWorkerEnabled::WhenValidating, service::Roles::AUTHORITY) => true,
+ (params::OffchainWorkerEnabled::Always, _) => true,
+ (params::OffchainWorkerEnabled::Never, _) => false,
+ (params::OffchainWorkerEnabled::WhenValidating, _) => false,
};
config.roles = role;
+ config.disable_grandpa = cli.no_grandpa;
+
let client_id = config.client_id();
fill_network_configuration(
cli.network_config,
@@ -387,7 +459,11 @@ where
}
if cli.shared_params.dev {
- config.keys.push("Alice".into());
+ config.keys.push("//Alice".into());
+ }
+
+ if let Some(account) = cli.keyring.account {
+ config.keys.push(format!("//{}", account));
}
let rpc_interface: &str = if cli.rpc_external { "0.0.0.0" } else { "127.0.0.1" };
@@ -407,6 +483,8 @@ where
config.telemetry_endpoints = Some(TelemetryEndpoints::new(cli.telemetry_endpoints));
}
+ config.force_authoring = cli.force_authoring;
+
Ok(config)
}
@@ -439,23 +517,19 @@ where
// 9926-9949 Unassigned
fn with_default_boot_node(
- mut spec: ChainSpec>,
- cli: &BuildSpecCmd,
+ spec: &mut ChainSpec>,
+ cli: BuildSpecCmd,
version: &VersionInfo,
-) -> error::Result>>
+) -> error::Result<()>
where
F: ServiceFactory
{
if spec.boot_nodes().is_empty() {
- let network_path =
- Some(network_path(&base_path(&cli.shared_params, version), spec.id()).to_string_lossy().into());
- let network_key = parse_node_key(cli.node_key.clone())?;
-
- let network_keys =
- network::obtain_private_key(&network_key, &network_path)
- .map_err(|err| format!("Error obtaining network key: {}", err))?;
-
- let peer_id = network_keys.to_peer_id();
+ let base_path = base_path(&cli.shared_params, version);
+ let storage_path = network_path(&base_path, spec.id());
+ let node_key = node_key_config(cli.node_key_params, &Some(storage_path))?;
+ let keys = node_key.into_keypair()?;
+ let peer_id = keys.public().into_peer_id();
let addr = build_multiaddr![
Ip4([127, 0, 0, 1]),
Tcp(30333u16),
@@ -463,7 +537,7 @@ where
];
spec.add_boot_node(addr)
}
- Ok(spec)
+ Ok(())
}
fn build_spec(
@@ -476,9 +550,10 @@ where
S: FnOnce(&str) -> Result>>, String>,
{
info!("Building chain spec");
- let spec = load_spec(&cli.shared_params, spec_factory)?;
- let spec = with_default_boot_node::(spec, &cli, version)?;
- let json = service::chain_ops::build_spec::>(spec, cli.raw)?;
+ let raw_output = cli.raw;
+ let mut spec = load_spec(&cli.shared_params, spec_factory)?;
+ with_default_boot_node::(&mut spec, cli, version)?;
+ let json = service::chain_ops::build_spec::>(spec, raw_output)?;
print!("{}", json);
@@ -574,23 +649,28 @@ where
S: FnOnce(&str) -> Result>>, String>,
{
let config = create_config_with_db_path::(spec_factory, &cli.shared_params, version)?;
-
let db_path = config.database_path;
- print!("Are you sure to remove {:?}? (y/n)", &db_path);
- stdout().flush().expect("failed to flush stdout");
-
- let mut input = String::new();
- stdin().read_line(&mut input)?;
- let input = input.trim();
-
- match input.chars().nth(0) {
- Some('y') | Some('Y') => {
- fs::remove_dir_all(&db_path)?;
- println!("{:?} removed.", &db_path);
- },
- _ => println!("Aborted"),
+
+ if cli.yes == false {
+ print!("Are you sure to remove {:?}? (y/n)", &db_path);
+ stdout().flush().expect("failed to flush stdout");
+
+ let mut input = String::new();
+ stdin().read_line(&mut input)?;
+ let input = input.trim();
+
+ match input.chars().nth(0) {
+ Some('y') | Some('Y') => {},
+ _ => {
+ println!("Aborted");
+ return Ok(());
+ },
+ }
}
+ fs::remove_dir_all(&db_path)?;
+ println!("{:?} removed.", &db_path);
+
Ok(())
}
@@ -698,6 +778,8 @@ fn kill_color(s: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
+ use tempdir::TempDir;
+ use network::identity::{secp256k1, ed25519};
#[test]
fn tests_node_name_good() {
@@ -713,4 +795,111 @@ mod tests {
assert!(is_node_name_valid("www.visit.me").is_err());
assert!(is_node_name_valid("email@domain").is_err());
}
+
+ #[test]
+ fn test_node_key_config_input() {
+ fn secret_input(net_config_dir: Option) -> error::Result<()> {
+ NodeKeyType::variants().into_iter().try_for_each(|t| {
+ let node_key_type = NodeKeyType::from_str(t).unwrap();
+ let sk = match node_key_type {
+ NodeKeyType::Secp256k1 => secp256k1::SecretKey::generate().as_ref().to_vec(),
+ NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec()
+ };
+ let params = NodeKeyParams {
+ node_key_type,
+ node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))),
+ node_key_file: None
+ };
+ node_key_config(params, &net_config_dir).and_then(|c| match c {
+ NodeKeyConfig::Secp256k1(network::Secret::Input(ref ski))
+ if node_key_type == NodeKeyType::Secp256k1 &&
+ &sk[..] == ski.as_ref() => Ok(()),
+ NodeKeyConfig::Ed25519(network::Secret::Input(ref ski))
+ if node_key_type == NodeKeyType::Ed25519 &&
+ &sk[..] == ski.as_ref() => Ok(()),
+ _ => Err(input_err("Unexpected node key config"))
+ })
+ })
+ }
+
+ assert!(secret_input(None).is_ok());
+ assert!(secret_input(Some("x".to_string())).is_ok());
+ }
+
+ #[test]
+ fn test_node_key_config_file() {
+ fn secret_file(net_config_dir: Option) -> error::Result<()> {
+ NodeKeyType::variants().into_iter().try_for_each(|t| {
+ let node_key_type = NodeKeyType::from_str(t).unwrap();
+ let tmp = TempDir::new("alice")?;
+ let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf();
+ let params = NodeKeyParams {
+ node_key_type,
+ node_key: None,
+ node_key_file: Some(file.clone())
+ };
+ node_key_config(params, &net_config_dir).and_then(|c| match c {
+ NodeKeyConfig::Secp256k1(network::Secret::File(ref f))
+ if node_key_type == NodeKeyType::Secp256k1 && f == &file => Ok(()),
+ NodeKeyConfig::Ed25519(network::Secret::File(ref f))
+ if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()),
+ _ => Err(input_err("Unexpected node key config"))
+ })
+ })
+ }
+
+ assert!(secret_file(None).is_ok());
+ assert!(secret_file(Some("x".to_string())).is_ok());
+ }
+
+ #[test]
+ fn test_node_key_config_default() {
+ fn with_def_params(f: F) -> error::Result<()>
+ where
+ F: Fn(NodeKeyParams) -> error::Result<()>
+ {
+ NodeKeyType::variants().into_iter().try_for_each(|t| {
+ let node_key_type = NodeKeyType::from_str(t).unwrap();
+ f(NodeKeyParams {
+ node_key_type,
+ node_key: None,
+ node_key_file: None
+ })
+ })
+ }
+
+ fn no_config_dir() -> error::Result<()> {
+ with_def_params(|params| {
+ let typ = params.node_key_type;
+ node_key_config::(params, &None)
+ .and_then(|c| match c {
+ NodeKeyConfig::Secp256k1(network::Secret::New)
+ if typ == NodeKeyType::Secp256k1 => Ok(()),
+ NodeKeyConfig::Ed25519(network::Secret::New)
+ if typ == NodeKeyType::Ed25519 => Ok(()),
+ _ => Err(input_err("Unexpected node key config"))
+ })
+ })
+ }
+
+ fn some_config_dir(net_config_dir: String) -> error::Result<()> {
+ with_def_params(|params| {
+ let dir = PathBuf::from(net_config_dir.clone());
+ let typ = params.node_key_type;
+ node_key_config(params, &Some(net_config_dir.clone()))
+ .and_then(move |c| match c {
+ NodeKeyConfig::Secp256k1(network::Secret::File(ref f))
+ if typ == NodeKeyType::Secp256k1 &&
+ f == &dir.join(NODE_KEY_SECP256K1_FILE) => Ok(()),
+ NodeKeyConfig::Ed25519(network::Secret::File(ref f))
+ if typ == NodeKeyType::Ed25519 &&
+ f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()),
+ _ => Err(input_err("Unexpected node key config"))
+ })
+ })
+ }
+
+ assert!(no_config_dir().is_ok());
+ assert!(some_config_dir("x".to_string()).is_ok());
+ }
}
diff --git a/core/cli/src/params.rs b/core/cli/src/params.rs
index 26a9a00ebf0a00d271b0036a2ff2958054962680..321cf17efb39cff59dd950dcd4d6b95df4cb7388 100644
--- a/core/cli/src/params.rs
+++ b/core/cli/src/params.rs
@@ -1,4 +1,4 @@
-// Copyright 2018 Parity Technologies (UK) Ltd.
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
use crate::traits::{AugmentClap, GetLogFilter};
use std::path::PathBuf;
-use structopt::{StructOpt, clap::{arg_enum, _clap_count_exprs, App, AppSettings, SubCommand}};
+use structopt::{StructOpt, clap::{arg_enum, _clap_count_exprs, App, AppSettings, SubCommand, Arg}};
use client;
/// Auxialary macro to implement `GetLogFilter` for all types that have the `shared_params` field.
@@ -39,6 +39,7 @@ arg_enum! {
Wasm,
Both,
NativeElseWasm,
+ NativeWhenPossible,
}
}
@@ -49,10 +50,21 @@ impl Into for ExecutionStrategy {
ExecutionStrategy::Wasm => client::ExecutionStrategy::AlwaysWasm,
ExecutionStrategy::Both => client::ExecutionStrategy::Both,
ExecutionStrategy::NativeElseWasm => client::ExecutionStrategy::NativeElseWasm,
+ ExecutionStrategy::NativeWhenPossible => client::ExecutionStrategy::NativeWhenPossible,
}
}
}
+arg_enum! {
+ /// How to execute blocks
+ #[derive(Debug, Clone)]
+ pub enum OffchainWorkerEnabled {
+ Always,
+ Never,
+ WhenValidating,
+ }
+}
+
/// Shared parameters used by all `CoreParams`.
#[derive(Debug, StructOpt, Clone)]
pub struct SharedParams {
@@ -68,7 +80,7 @@ pub struct SharedParams {
#[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))]
pub base_path: Option,
- ///Sets a custom logging filter
+ /// Sets a custom logging filter
#[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")]
pub log: Option,
}
@@ -98,10 +110,6 @@ pub struct NetworkConfigurationParams {
#[structopt(long = "port", value_name = "PORT")]
pub port: Option,
- /// Specify node secret key (64-character hex string)
- #[structopt(long = "node-key", value_name = "KEY")]
- pub node_key: Option,
-
/// Specify the number of outgoing connections we're trying to maintain
#[structopt(long = "out-peers", value_name = "OUT_PEERS", default_value = "25")]
pub out_peers: u32,
@@ -109,6 +117,98 @@ pub struct NetworkConfigurationParams {
/// Specify the maximum number of incoming connections we're accepting
#[structopt(long = "in-peers", value_name = "IN_PEERS", default_value = "25")]
pub in_peers: u32,
+
+ /// By default, the network will use mDNS to discover other nodes on the local network. This
+ /// disables it.
+ #[structopt(long = "no-mdns")]
+ pub no_mdns: bool,
+
+ #[allow(missing_docs)]
+ #[structopt(flatten)]
+ pub node_key_params: NodeKeyParams
+}
+
+arg_enum! {
+ #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+ pub enum NodeKeyType {
+ Secp256k1,
+ Ed25519
+ }
+}
+
+/// Parameters used to create the `NodeKeyConfig`, which determines the keypair
+/// used for libp2p networking.
+#[derive(Debug, StructOpt, Clone)]
+pub struct NodeKeyParams {
+ /// The secret key to use for libp2p networking.
+ ///
+ /// The value is a string that is parsed according to the choice of
+ /// `--node-key-type` as follows:
+ ///
+ /// `secp256k1`:
+ /// The value is parsed as a hex-encoded Secp256k1 32 bytes secret key,
+ /// i.e. 64 hex characters.
+ ///
+ /// `ed25519`:
+ /// The value is parsed as a hex-encoded Ed25519 32 bytes secret key,
+ /// i.e. 64 hex characters.
+ ///
+ /// The value of this option takes precedence over `--node-key-file`.
+ ///
+ /// WARNING: Secrets provided as command-line arguments are easily exposed.
+ /// Use of this option should be limited to development and testing. To use
+ /// an externally managed secret key, use `--node-key-file` instead.
+ #[structopt(long = "node-key", value_name = "KEY")]
+ pub node_key: Option,
+
+ /// The type of secret key to use for libp2p networking.
+ ///
+ /// The secret key of the node is obtained as follows:
+ ///
+ /// * If the `--node-key` option is given, the value is parsed as a secret key
+ /// according to the type. See the documentation for `--node-key`.
+ ///
+ /// * If the `--node-key-file` option is given, the secret key is read from the
+ /// specified file. See the documentation for `--node-key-file`.
+ ///
+ /// * Otherwise, the secret key is read from a file with a predetermined,
+ /// type-specific name from the chain-specific network config directory
+ /// inside the base directory specified by `--base-dir`. If this file does
+ /// not exist, it is created with a newly generated secret key of the
+ /// chosen type.
+ ///
+ /// The node's secret key determines the corresponding public key and hence the
+ /// node's peer ID in the context of libp2p.
+ ///
+ /// NOTE: The current default key type is `secp256k1` for a transition period only
+ /// but will eventually change to `ed25519` in a future release. To continue using
+ /// `secp256k1` keys, use `--node-key-type=secp256k1`.
+ #[structopt(
+ long = "node-key-type",
+ value_name = "TYPE",
+ raw(
+ possible_values = "&NodeKeyType::variants()",
+ case_insensitive = "true",
+ default_value = r#""Secp256k1""#
+ )
+ )]
+ pub node_key_type: NodeKeyType,
+
+ /// The file from which to read the node's secret key to use for libp2p networking.
+ ///
+ /// The contents of the file are parsed according to the choice of `--node-key-type`
+ /// as follows:
+ ///
+ /// `secp256k1`:
+ /// The file must contain an unencoded 32 bytes Secp256k1 secret key.
+ ///
+ /// `ed25519`:
+ /// The file must contain an unencoded 32 bytes Ed25519 secret key.
+ ///
+ /// If the file does not exist, it is created with a newly generated secret key of
+ /// the chosen type.
+ #[structopt(long = "node-key-file", value_name = "FILE")]
+ pub node_key_file: Option
}
/// Parameters used to create the pool configuration.
@@ -122,6 +222,70 @@ pub struct TransactionPoolParams {
pub pool_kbytes: usize,
}
+/// Execution strategies parameters.
+#[derive(Debug, StructOpt, Clone)]
+pub struct ExecutionStrategies {
+ /// The means of execution used when calling into the runtime while syncing blocks.
+ #[structopt(
+ long = "syncing-execution",
+ value_name = "STRATEGY",
+ raw(
+ possible_values = "&ExecutionStrategy::variants()",
+ case_insensitive = "true",
+ default_value = r#""NativeElseWasm""#
+ )
+ )]
+ pub syncing_execution: ExecutionStrategy,
+
+ /// The means of execution used when calling into the runtime while importing blocks.
+ #[structopt(
+ long = "importing-execution",
+ value_name = "STRATEGY",
+ raw(
+ possible_values = "&ExecutionStrategy::variants()",
+ case_insensitive = "true",
+ default_value = r#""NativeElseWasm""#
+ )
+ )]
+ pub importing_execution: ExecutionStrategy,
+
+ /// The means of execution used when calling into the runtime while constructing blocks.
+ #[structopt(
+ long = "block-construction-execution",
+ value_name = "STRATEGY",
+ raw(
+ possible_values = "&ExecutionStrategy::variants()",
+ case_insensitive = "true",
+ default_value = r#""Wasm""#
+ )
+ )]
+ pub block_construction_execution: ExecutionStrategy,
+
+ /// The means of execution used when calling into the runtime while constructing blocks.
+ #[structopt(
+ long = "offchain-worker-execution",
+ value_name = "STRATEGY",
+ raw(
+ possible_values = "&ExecutionStrategy::variants()",
+ case_insensitive = "true",
+ default_value = r#""NativeWhenPossible""#
+ )
+ )]
+ pub offchain_worker_execution: ExecutionStrategy,
+
+ /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks.
+ #[structopt(
+ long = "other-execution",
+ value_name = "STRATEGY",
+ raw(
+ possible_values = "&ExecutionStrategy::variants()",
+ case_insensitive = "true",
+ default_value = r#""Wasm""#
+ )
+ )]
+ pub other_execution: ExecutionStrategy,
+}
+
/// The `run` command used to run a node.
#[derive(Debug, StructOpt, Clone)]
pub struct RunCmd {
@@ -137,6 +301,10 @@ pub struct RunCmd {
#[structopt(long = "validator")]
pub validator: bool,
+ /// Disable GRANDPA when running in validator mode
+ #[structopt(long = "no-grandpa")]
+ pub no_grandpa: bool,
+
/// Run in light client mode
#[structopt(long = "light")]
pub light: bool,
@@ -169,7 +337,7 @@ pub struct RunCmd {
#[structopt(long = "name", value_name = "NAME")]
pub name: Option,
- /// Should not connect to the Substrate telemetry server (telemetry is on by default on global chains)
+ /// Disable connecting to the Substrate telemetry server (telemetry is on by default on global chains).
#[structopt(long = "no-telemetry")]
pub no_telemetry: bool,
@@ -179,55 +347,23 @@ pub struct RunCmd {
#[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = "parse_telemetry_endpoints"))]
pub telemetry_endpoints: Vec<(String, u8)>,
- /// The means of execution used when calling into the runtime while syncing blocks.
+ /// Should execute offchain workers on every block. By default it's only enabled for nodes that are authoring new
+ /// blocks.
#[structopt(
- long = "syncing-execution",
- value_name = "STRATEGY",
+ long = "offchain-worker",
+ value_name = "ENABLED",
raw(
- possible_values = "&ExecutionStrategy::variants()",
+ possible_values = "&OffchainWorkerEnabled::variants()",
case_insensitive = "true",
- default_value = r#""NativeElseWasm""#
+ default_value = r#""WhenValidating""#
)
)]
- pub syncing_execution: ExecutionStrategy,
+ pub offchain_worker: OffchainWorkerEnabled,
- /// The means of execution used when calling into the runtime while importing blocks.
- #[structopt(
- long = "importing-execution",
- value_name = "STRATEGY",
- raw(
- possible_values = "&ExecutionStrategy::variants()",
- case_insensitive = "true",
- default_value = r#""NativeElseWasm""#
- )
- )]
- pub importing_execution: ExecutionStrategy,
-
- /// The means of execution used when calling into the runtime while constructing blocks.
- #[structopt(
- long = "block-construction-execution",
- value_name = "STRATEGY",
- raw(
- possible_values = "&ExecutionStrategy::variants()",
- case_insensitive = "true",
- default_value = r#""Wasm""#
- )
- )]
- pub block_construction_execution: ExecutionStrategy,
-
- /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks.
- #[structopt(
- long = "other-execution",
- value_name = "STRATEGY",
- raw(
- possible_values = "&ExecutionStrategy::variants()",
- case_insensitive = "true",
- default_value = r#""Wasm""#
- )
- )]
- pub other_execution: ExecutionStrategy,
+ #[allow(missing_docs)]
+ #[structopt(flatten)]
+ pub execution_strategies: ExecutionStrategies,
-
#[allow(missing_docs)]
#[structopt(flatten)]
pub shared_params: SharedParams,
@@ -239,6 +375,84 @@ pub struct RunCmd {
#[allow(missing_docs)]
#[structopt(flatten)]
pub pool_config: TransactionPoolParams,
+
+ #[allow(missing_docs)]
+ #[structopt(flatten)]
+ pub keyring: Keyring,
+
+ /// Enable authoring even when offline.
+ #[structopt(long = "force-authoring")]
+ pub force_authoring: bool,
+}
+
+/// Stores all required Cli values for a keyring test account.
+struct KeyringTestAccountCliValues {
+ help: String,
+ conflicts_with: Vec,
+ name: String,
+ variant: keyring::AuthorityKeyring,
+}
+
+lazy_static::lazy_static! {
+ /// The Cli values for all test accounts.
+ static ref TEST_ACCOUNTS_CLI_VALUES: Vec = {
+ keyring::AuthorityKeyring::iter().map(|a| {
+ let help = format!("Shortcut for `--key //{} --name {}`.", a, a);
+ let conflicts_with = keyring::AuthorityKeyring::iter()
+ .filter(|b| a != *b)
+ .map(|b| b.to_string().to_lowercase())
+ .chain(["name", "key"].iter().map(|s| s.to_string()))
+ .collect::>();
+ let name = a.to_string().to_lowercase();
+
+ KeyringTestAccountCliValues {
+ help,
+ conflicts_with,
+ name,
+ variant: a,
+ }
+ }).collect()
+ };
+}
+
+/// Wrapper for exposing the keyring test accounts into the Cli.
+#[derive(Debug, Clone)]
+pub struct Keyring {
+ pub account: Option,
+}
+
+impl StructOpt for Keyring {
+ fn clap<'a, 'b>() -> App<'a, 'b> {
+ unimplemented!("Should not be called for `TestAccounts`.")
+ }
+
+ fn from_clap(m: &::structopt::clap::ArgMatches) -> Self {
+ Keyring {
+ account: TEST_ACCOUNTS_CLI_VALUES.iter().find(|a| m.is_present(&a.name)).map(|a| a.variant),
+ }
+ }
+}
+
+impl AugmentClap for Keyring {
+ fn augment_clap<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
+ TEST_ACCOUNTS_CLI_VALUES.iter().fold(app, |app, a| {
+ let conflicts_with_strs = a.conflicts_with.iter().map(|s| s.as_str()).collect::>();
+
+ app.arg(
+ Arg::with_name(&a.name)
+ .long(&a.name)
+ .help(&a.help)
+ .conflicts_with_all(&conflicts_with_strs)
+ .takes_value(false)
+ )
+ })
+ }
+}
+
+impl Keyring {
+ fn is_subcommand() -> bool {
+ false
+ }
}
/// Default to verbosity level 0, if none is provided.
@@ -270,9 +484,9 @@ pub struct BuildSpecCmd {
#[structopt(flatten)]
pub shared_params: SharedParams,
- /// Specify node secret key (64-character hex string)
- #[structopt(long = "node-key", value_name = "KEY")]
- pub node_key: Option,
+ #[allow(missing_docs)]
+ #[structopt(flatten)]
+ pub node_key_params: NodeKeyParams,
}
impl_get_log_filter!(BuildSpecCmd);
@@ -338,6 +552,10 @@ impl_get_log_filter!(RevertCmd);
/// The `purge-chain` command used to remove the whole chain.
#[derive(Debug, StructOpt, Clone)]
pub struct PurgeChainCmd {
+ /// Skip interactive prompt by answering yes automatically.
+ #[structopt(short = "y")]
+ pub yes: bool,
+
#[allow(missing_docs)]
#[structopt(flatten)]
pub shared_params: SharedParams,
diff --git a/core/client/Cargo.toml b/core/client/Cargo.toml
index 495b11b61d5b8aa0ecda2a9507b608db280b254d..cfc7eb3e3447e4f4c2a39d46fb5dcbadac9bdeef 100644
--- a/core/client/Cargo.toml
+++ b/core/client/Cargo.toml
@@ -18,9 +18,9 @@ state-machine = { package = "substrate-state-machine", path = "../state-machine"
keyring = { package = "substrate-keyring", path = "../keyring", optional = true }
trie = { package = "substrate-trie", path = "../trie", optional = true }
substrate-telemetry = { path = "../telemetry", optional = true }
-hash-db = { version = "0.11", optional = true }
+hash-db = { version = "0.12", optional = true }
kvdb = { git = "https://github.com/paritytech/parity-common", optional = true, rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
-parity-codec = { version = "3.1", default-features = false, features = ["derive"] }
+parity-codec = { version = "3.2", default-features = false, features = ["derive"] }
primitives = { package = "substrate-primitives", path = "../primitives", default-features = false }
runtime-primitives = { package = "sr-primitives", path = "../sr-primitives", default-features = false }
runtime-version = { package = "sr-version", path = "../sr-version", default-features = false }
diff --git a/core/client/db/Cargo.toml b/core/client/db/Cargo.toml
index 8828eefa4fbf391d21bb4d8fbc3ef2396461c486..2944ab533b551c924d55c7fbf2adc81b36b5857b 100644
--- a/core/client/db/Cargo.toml
+++ b/core/client/db/Cargo.toml
@@ -12,15 +12,16 @@ kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c
kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
lru-cache = "0.1.1"
-hash-db = { version = "0.11" }
+hash-db = { version = "0.12" }
primitives = { package = "substrate-primitives", path = "../../primitives" }
runtime_primitives = { package = "sr-primitives", path = "../../sr-primitives" }
client = { package = "substrate-client", path = "../../client" }
state-machine = { package = "substrate-state-machine", path = "../../state-machine" }
-parity-codec = { version = "3.1", features = ["derive"] }
+parity-codec = { version = "3.2", features = ["derive"] }
executor = { package = "substrate-executor", path = "../../executor" }
state_db = { package = "substrate-state-db", path = "../../state-db" }
trie = { package = "substrate-trie", path = "../../trie" }
+consensus_common = { package = "substrate-consensus-common", path = "../../consensus/common" }
[dev-dependencies]
kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
diff --git a/core/client/db/src/cache/list_cache.rs b/core/client/db/src/cache/list_cache.rs
index f8e52e14c630f07b421eaff4b69ca1bd0071c4fc..1e641534f969c0d63eefe387e98c9ded8bb3c81a 100644
--- a/core/client/db/src/cache/list_cache.rs
+++ b/core/client/db/src/cache/list_cache.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
diff --git a/core/client/db/src/cache/list_entry.rs b/core/client/db/src/cache/list_entry.rs
index d2d88ab12f38ab341a6b69b51d35fbe989ac23e5..237ae9a268026744b87f6d85b7c6301cb82ee204 100644
--- a/core/client/db/src/cache/list_entry.rs
+++ b/core/client/db/src/cache/list_entry.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
diff --git a/core/client/db/src/cache/list_storage.rs b/core/client/db/src/cache/list_storage.rs
index b9bf99f67b884fba4ae85f358cb9c0b731cb834b..659a30507e136576a844624fa167eb1cfb08ddb9 100644
--- a/core/client/db/src/cache/list_storage.rs
+++ b/core/client/db/src/cache/list_storage.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
diff --git a/core/client/db/src/cache/mod.rs b/core/client/db/src/cache/mod.rs
index 12f8c5e519782a9b2db3e7842176665e41b120f5..b5dd45f11dd52e69ea715243924ac5e799e038b1 100644
--- a/core/client/db/src/cache/mod.rs
+++ b/core/client/db/src/cache/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
//! DB-backed cache of blockchain data.
-use std::sync::Arc;
+use std::{sync::Arc, collections::HashMap};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -25,7 +25,8 @@ use client::blockchain::Cache as BlockchainCache;
use client::error::Result as ClientResult;
use parity_codec::{Encode, Decode};
use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, As, AuthorityIdFor};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, As};
+use consensus_common::well_known_cache_keys::Id as CacheKeyId;
use crate::utils::{self, COLUMN_META};
use self::list_cache::ListCache;
@@ -64,7 +65,12 @@ impl CacheItemT for T where T: Clone + Decode + Encode + PartialEq {}
/// Database-backed blockchain data cache.
pub struct DbCache {
- authorities_at: ListCache>, self::list_storage::DbStorage>,
+ cache_at: HashMap, self::list_storage::DbStorage>>,
+ db: Arc,
+ key_lookup_column: Option,
+ header_column: Option,
+ authorities_column: Option,
+ best_finalized_block: ComplexBlockId,
}
impl DbCache {
@@ -76,19 +82,13 @@ impl DbCache {
authorities_column: Option,
best_finalized_block: ComplexBlockId,
) -> Self {
- DbCache {
- authorities_at: ListCache::new(
- self::list_storage::DbStorage::new(b"auth".to_vec(), db,
- self::list_storage::DbColumns {
- meta: COLUMN_META,
- key_lookup: key_lookup_column,
- header: header_column,
- cache: authorities_column,
- },
- ),
- As::sa(PRUNE_DEPTH),
- best_finalized_block,
- ),
+ Self {
+ cache_at: HashMap::new(),
+ db,
+ key_lookup_column,
+ header_column,
+ authorities_column,
+ best_finalized_block,
}
}
@@ -97,35 +97,82 @@ impl DbCache {
DbCacheTransaction {
cache: self,
tx,
- authorities_at_op: None,
+ cache_at_op: HashMap::new(),
+ best_finalized_block: None,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps) {
- if let Some(authorities_at_op) = ops.authorities_at_op {
- self.authorities_at.on_transaction_commit(authorities_at_op);
+ for (name, op) in ops.cache_at_op.into_iter() {
+ self.get_cache(name).on_transaction_commit(op);
+ }
+ if let Some(best_finalized_block) = ops.best_finalized_block {
+ self.best_finalized_block = best_finalized_block;
}
}
+
+ /// Creates `ListCache` with the given name or returns a reference to the existing.
+ fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache, self::list_storage::DbStorage> {
+ get_cache_helper(
+ &mut self.cache_at,
+ name,
+ &self.db,
+ self.key_lookup_column,
+ self.header_column,
+ self.authorities_column,
+ &self.best_finalized_block
+ )
+ }
+}
+
+// This helper is needed because otherwise the borrow checker will require to
+// clone all parameters outside of the closure.
+fn get_cache_helper<'a, Block: BlockT>(
+ cache_at: &'a mut HashMap, self::list_storage::DbStorage>>,
+ name: CacheKeyId,
+ db: &Arc,
+ key_lookup: Option,
+ header: Option,
+ cache: Option,
+ best_finalized_block: &ComplexBlockId,
+) -> &'a mut ListCache, self::list_storage::DbStorage> {
+ cache_at.entry(name).or_insert_with(|| {
+ ListCache::new(
+ self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
+ self::list_storage::DbColumns {
+ meta: COLUMN_META,
+ key_lookup,
+ header,
+ cache,
+ },
+ ),
+ As::sa(PRUNE_DEPTH),
+ best_finalized_block.clone(),
+ )
+ })
}
/// Cache operations that are to be committed after database transaction is committed.
pub struct DbCacheTransactionOps {
- authorities_at_op: Option>>>,
+ cache_at_op: HashMap>>,
+ best_finalized_block: Option>,
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache,
tx: &'a mut DBTransaction,
- authorities_at_op: Option>>>,
+ cache_at_op: HashMap>>,
+ best_finalized_block: Option>,
}
impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps {
DbCacheTransactionOps {
- authorities_at_op: self.authorities_at_op,
+ cache_at_op: self.cache_at_op,
+ best_finalized_block: self.best_finalized_block,
}
}
@@ -134,21 +181,42 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
mut self,
parent: ComplexBlockId,
block: ComplexBlockId,
- authorities_at: Option>>,
+ data_at: HashMap>,
is_final: bool,
) -> ClientResult {
- assert!(self.authorities_at_op.is_none());
+ assert!(self.cache_at_op.is_empty());
- self.authorities_at_op = self.cache.authorities_at.on_block_insert(
- &mut self::list_storage::DbStorageTransaction::new(
- self.cache.authorities_at.storage(),
- &mut self.tx
- ),
- parent,
- block,
- authorities_at,
- is_final,
- )?;
+ // prepare list of caches that are not update
+ // (we might still need to do some cache maintenance in this case)
+ let missed_caches = self.cache.cache_at.keys()
+ .filter(|cache| !data_at.contains_key(cache.clone()))
+ .cloned()
+ .collect::>();
+
+ let mut insert_op = |name: CacheKeyId, value: Option>| -> Result<(), client::error::Error> {
+ let cache = self.cache.get_cache(name);
+ let op = cache.on_block_insert(
+ &mut self::list_storage::DbStorageTransaction::new(
+ cache.storage(),
+ &mut self.tx,
+ ),
+ parent.clone(),
+ block.clone(),
+ value.or(cache.value_at_block(&parent)?),
+ is_final,
+ )?;
+ if let Some(op) = op {
+ self.cache_at_op.insert(name, op);
+ }
+ Ok(())
+ };
+
+ data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?;
+ missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
+
+ if is_final {
+ self.best_finalized_block = Some(block);
+ }
Ok(self)
}
@@ -159,16 +227,24 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
parent: ComplexBlockId,
block: ComplexBlockId
) -> ClientResult {
- assert!(self.authorities_at_op.is_none());
+ assert!(self.cache_at_op.is_empty());
- self.authorities_at_op = self.cache.authorities_at.on_block_finalize(
- &mut self::list_storage::DbStorageTransaction::new(
- self.cache.authorities_at.storage(),
- &mut self.tx
- ),
- parent,
- block,
- )?;
+ for (name, cache_at) in self.cache.cache_at.iter() {
+ let op = cache_at.on_block_finalize(
+ &mut self::list_storage::DbStorageTransaction::new(
+ cache_at.storage(),
+ &mut self.tx
+ ),
+ parent.clone(),
+ block.clone(),
+ )?;
+
+ if let Some(op) = op {
+ self.cache_at_op.insert(name.to_owned(), op);
+ }
+ }
+
+ self.best_finalized_block = Some(block);
Ok(self)
}
@@ -178,12 +254,12 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
pub struct DbCacheSync(pub RwLock>);
impl BlockchainCache for DbCacheSync {
- fn authorities_at(&self, at: BlockId) -> Option>> {
+ fn get_at(&self, key: &CacheKeyId, at: &BlockId) -> Option> {
let cache = self.0.read();
- let storage = cache.authorities_at.storage();
+ let storage = cache.cache_at.get(key)?.storage();
let db = storage.db();
let columns = storage.columns();
- let at = match at {
+ let at = match *at {
BlockId::Hash(hash) => {
let header = utils::read_header::(
&**db,
@@ -202,6 +278,7 @@ impl BlockchainCache for DbCacheSync {
},
};
- cache.authorities_at.value_at_block(&at).ok()?
+ cache.cache_at.get(key)?.value_at_block(&at).ok()?
}
}
+
diff --git a/core/client/db/src/lib.rs b/core/client/db/src/lib.rs
index 36ac3cc3450552cc8e6f4280898ca413a8fb3ec3..7657bfd39691a0e2bec1e89e8ed7d84339ae1446 100644
--- a/core/client/db/src/lib.rs
+++ b/core/client/db/src/lib.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -33,6 +33,7 @@ mod utils;
use std::sync::Arc;
use std::path::PathBuf;
use std::io;
+use std::collections::HashMap;
use client::backend::NewBlockState;
use client::blockchain::HeaderBackend;
@@ -40,12 +41,12 @@ use client::ExecutionStrategies;
use parity_codec::{Decode, Encode};
use hash_db::Hasher;
use kvdb::{KeyValueDB, DBTransaction};
-use trie::MemoryDB;
+use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
use parking_lot::RwLock;
use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash};
use primitives::storage::well_known_keys;
use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero, Digest, DigestItem, AuthorityIdFor};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero, Digest, DigestItem};
use runtime_primitives::BuildStorage;
use state_machine::backend::Backend as StateBackend;
use executor::RuntimeInfo;
@@ -54,6 +55,7 @@ use crate::utils::{Meta, db_err, meta_keys, open_database, read_db, block_id_to_
use client::leaves::{LeafSet, FinalizationDisplaced};
use client::children;
use state_db::StateDb;
+use consensus_common::well_known_cache_keys;
use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
use log::{trace, debug, warn};
pub use state_db::PruningMode;
@@ -243,7 +245,7 @@ impl client::blockchain::Backend for BlockchainDb {
Ok(self.meta.read().finalized_hash.clone())
}
- fn cache(&self) -> Option<&client::blockchain::Cache> {
+ fn cache(&self) -> Option>> {
None
}
@@ -256,10 +258,16 @@ impl client::blockchain::Backend for BlockchainDb {
}
}
+impl client::blockchain::ProvideCache for BlockchainDb {
+ fn cache(&self) -> Option>> {
+ None
+ }
+}
+
/// Database transaction
pub struct BlockImportOperation {
old_state: CachingState,
- db_updates: MemoryDB,
+ db_updates: PrefixedMemoryDB,
storage_updates: Vec<(Vec, Option>)>,
changes_trie_updates: MemoryDB,
pending_block: Option>,
@@ -306,11 +314,11 @@ where Block: BlockT,
Ok(())
}
- fn update_authorities(&mut self, _authorities: Vec>) {
- // currently authorities are not cached on full nodes
+ fn update_cache(&mut self, _cache: HashMap>) {
+ // Currently cache isn't implemented on full nodes.
}
- fn update_db_storage(&mut self, update: MemoryDB) -> Result<(), client::error::Error> {
+ fn update_db_storage(&mut self, update: PrefixedMemoryDB) -> Result<(), client::error::Error> {
self.db_updates = update;
Ok(())
}
@@ -321,7 +329,7 @@ where Block: BlockT,
return Err(client::error::ErrorKind::GenesisInvalid.into());
}
- let mut transaction: MemoryDB = Default::default();
+ let mut transaction: PrefixedMemoryDB = Default::default();
for (child_key, child_map) in children {
if !well_known_keys::is_child_storage_key(&child_key) {
@@ -374,22 +382,23 @@ where Block: BlockT,
struct StorageDb {
pub db: Arc,
- pub state_db: StateDb,
+ pub state_db: StateDb>,
}
impl state_machine::Storage for StorageDb {
- fn get(&self, key: &H256) -> Result, String> {
- self.state_db.get(key, self).map(|r| r.map(|v| DBValue::from_slice(&v)))
+ fn get(&self, key: &H256, prefix: &[u8]) -> Result, String> {
+ let key = prefixed_key::(key, prefix);
+ self.state_db.get(&key, self).map(|r| r.map(|v| DBValue::from_slice(&v)))
.map_err(|e| format!("Database backend error: {:?}", e))
}
}
-impl state_db::HashDb for StorageDb {
+impl state_db::NodeDb for StorageDb {
type Error = io::Error;
- type Hash = H256;
+ type Key = [u8];
- fn get(&self, key: &H256) -> Result>, Self::Error> {
- self.db.get(columns::STATE, key.as_bytes()).map(|r| r.map(|v| v.to_vec()))
+ fn get(&self, key: &[u8]) -> Result>, Self::Error> {
+ self.db.get(columns::STATE, key).map(|r| r.map(|v| v.to_vec()))
}
}
@@ -405,7 +414,7 @@ impl DbGenesisStorage {
}
impl state_machine::Storage for DbGenesisStorage {
- fn get(&self, _key: &H256) -> Result, String> {
+ fn get(&self, _key: &H256, _prefix: &[u8]) -> Result, String> {
Ok(None)
}
}
@@ -515,7 +524,7 @@ impl state_machine::ChangesTrieRootsStorage for DbC
}
impl state_machine::ChangesTrieStorage for DbChangesTrieStorage {
- fn get(&self, key: &H256) -> Result, String> {
+ fn get(&self, key: &H256, _prefix: &[u8]) -> Result, String> {
self.db.get(columns::CHANGES_TRIE, &key[..])
.map_err(|err| format!("{}", err))
}
@@ -559,7 +568,7 @@ impl> Backend {
let blockchain = BlockchainDb::new(db.clone())?;
let meta = blockchain.meta.clone();
let map_e = |e: state_db::Error| ::client::error::Error::from(format!("State database error: {:?}", e));
- let state_db: StateDb = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?;
+ let state_db: StateDb<_, _> = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?;
let storage_db = StorageDb {
db: db.clone(),
state_db,
@@ -832,7 +841,7 @@ impl> Backend {
transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref());
}
- let mut changeset: state_db::ChangeSet = state_db::ChangeSet::default();
+ let mut changeset: state_db::ChangeSet> = state_db::ChangeSet::default();
for (key, (val, rc)) in operation.db_updates.drain() {
if rc > 0 {
changeset.inserted.push((key, val.to_vec()));
@@ -981,7 +990,7 @@ impl> Backend {
}
}
-fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet) {
+fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet>) {
for (key, val) in commit.data.inserted.into_iter() {
transaction.put(columns::STATE, &key[..], &val);
}
@@ -1031,7 +1040,7 @@ impl client::backend::Backend for Backend whe
Ok(BlockImportOperation {
pending_block: None,
old_state,
- db_updates: MemoryDB::default(),
+ db_updates: PrefixedMemoryDB::default(),
storage_updates: Default::default(),
changes_trie_updates: MemoryDB::default(),
aux_ops: Vec::new(),
@@ -1169,6 +1178,10 @@ impl client::backend::Backend for Backend whe
}
}
+ fn have_state_at(&self, hash: &Block::Hash, number: NumberFor) -> bool {
+ !self.storage.state_db.is_pruned(hash, number.as_())
+ }
+
fn destroy_state(&self, mut state: Self::State) -> Result<(), client::error::Error> {
if let Some(hash) = state.parent_hash.clone() {
let is_best = || self.blockchain.meta.read().best_hash == hash;
@@ -1410,7 +1423,7 @@ mod tests {
op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap();
- key = op.db_updates.insert(b"hello");
+ key = op.db_updates.insert(&[], b"hello");
op.set_block_data(
header,
Some(vec![]),
@@ -1444,8 +1457,8 @@ mod tests {
).0.into();
let hash = header.hash();
- op.db_updates.insert(b"hello");
- op.db_updates.remove(&key);
+ op.db_updates.insert(&[], b"hello");
+ op.db_updates.remove(&key, &[]);
op.set_block_data(
header,
Some(vec![]),
@@ -1479,7 +1492,7 @@ mod tests {
).0.into();
let hash = header.hash();
- op.db_updates.remove(&key);
+ op.db_updates.remove(&key, &[]);
op.set_block_data(
header,
Some(vec![]),
@@ -1545,7 +1558,7 @@ mod tests {
assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root)));
for (key, (val, _)) in changes_trie_update.drain() {
- assert_eq!(backend.changes_trie_storage().unwrap().get(&key), Ok(Some(val)));
+ assert_eq!(backend.changes_trie_storage().unwrap().get(&key, &[]), Ok(Some(val)));
}
};
@@ -1671,23 +1684,23 @@ mod tests {
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(Some(config.clone()), &mut tx, Default::default(), 12);
backend.storage.db.write(tx).unwrap();
- assert!(backend.changes_tries_storage.get(&root1).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root2).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root3).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root4).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root5).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root6).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root7).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root8).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root4, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_some());
// now simulate finalization of block#16, causing prune of tries at #5..#8
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(Some(config.clone()), &mut tx, Default::default(), 16);
backend.storage.db.write(tx).unwrap();
- assert!(backend.changes_tries_storage.get(&root5).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root6).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root7).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root8).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_none());
// now "change" pruning mode to archive && simulate finalization of block#20
// => no changes tries are pruned, because we never prune in archive mode
@@ -1695,10 +1708,10 @@ mod tests {
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(Some(config), &mut tx, Default::default(), 20);
backend.storage.db.write(tx).unwrap();
- assert!(backend.changes_tries_storage.get(&root9).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root10).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root11).unwrap().is_some());
- assert!(backend.changes_tries_storage.get(&root12).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root9, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root10, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root11, &[]).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root12, &[]).unwrap().is_some());
}
#[test]
@@ -1737,15 +1750,15 @@ mod tests {
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(Some(config.clone()), &mut tx, block5, 5);
backend.storage.db.write(tx).unwrap();
- assert!(backend.changes_tries_storage.get(&root1).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root2).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_some());
// now simulate finalization of block#6, causing prune of tries at #2
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(Some(config.clone()), &mut tx, block6, 6);
backend.storage.db.write(tx).unwrap();
- assert!(backend.changes_tries_storage.get(&root2).unwrap().is_none());
- assert!(backend.changes_tries_storage.get(&root3).unwrap().is_some());
+ assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none());
+ assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_some());
}
#[test]
diff --git a/core/client/db/src/light.rs b/core/client/db/src/light.rs
index 8a3456df69a305c55870617c4147950a517c0da5..62b6486f54d7ea28ac5561a33aa5b374943e26ec 100644
--- a/core/client/db/src/light.rs
+++ b/core/client/db/src/light.rs
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
//! RocksDB-based light client blockchain storage.
-use std::sync::Arc;
+use std::{sync::Arc, collections::HashMap};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -32,7 +32,8 @@ use parity_codec::{Decode, Encode};
use primitives::Blake2Hasher;
use runtime_primitives::generic::BlockId;
use runtime_primitives::traits::{Block as BlockT, Header as HeaderT,
- Zero, One, As, NumberFor, Digest, DigestItem, AuthorityIdFor};
+ Zero, One, As, NumberFor, Digest, DigestItem};
+use consensus_common::well_known_cache_keys;
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId};
use crate::utils::{self, meta_keys, Meta, db_err, open_database,
read_db, block_id_to_lookup_key, read_meta};
@@ -59,7 +60,7 @@ pub struct LightStorage {
db: Arc,
meta: RwLock, Block::Hash>>,
leaves: RwLock>>,
- cache: DbCacheSync,
+ cache: Arc>,
}
impl LightStorage
@@ -96,7 +97,7 @@ impl LightStorage
Ok(LightStorage {
db,
meta: RwLock::new(meta),
- cache: DbCacheSync(RwLock::new(cache)),
+ cache: Arc::new(DbCacheSync(RwLock::new(cache))),
leaves: RwLock::new(leaves),
})
}
@@ -370,7 +371,7 @@ impl LightBlockchainStorage for LightStorage
fn import_header(
&self,
header: Block::Header,
- authorities: Option>>,
+ cache_at: HashMap>,
leaf_state: NewBlockState,
aux_ops: Vec<(Vec, Option>)>,
) -> ClientResult<()> {
@@ -432,7 +433,7 @@ impl LightBlockchainStorage for LightStorage
.on_block_insert(
ComplexBlockId::new(*header.parent_hash(), if number.is_zero() { Zero::zero() } else { number - One::one() }),
ComplexBlockId::new(hash, number),
- authorities,
+ cache_at,
finalized,
)?
.into_ops();
@@ -521,8 +522,8 @@ impl LightBlockchainStorage for LightStorage
Ok(self.meta.read().finalized_hash.clone())
}
- fn cache(&self) -> Option<&BlockchainCache> {
- Some(&self.cache)
+ fn cache(&self) -> Option>> {
+ Some(self.cache.clone())
}
}
@@ -538,9 +539,11 @@ pub(crate) mod tests {
use client::cht;
use runtime_primitives::generic::DigestItem;
use runtime_primitives::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
+ use runtime_primitives::traits::AuthorityIdFor;
use super::*;
type Block = RawBlock>;
+ type AuthorityId = AuthorityIdFor;
pub fn default_header(parent: &Hash, number: u64) -> Header {
Header {
@@ -566,41 +569,41 @@ pub(crate) mod tests {
pub fn insert_block Header>(
db: &LightStorage,
- authorities: Option>>,
+ cache: HashMap>,
header: F,
) -> Hash {
let header = header();
let hash = header.hash();
- db.import_header(header, authorities, NewBlockState::Best, Vec::new()).unwrap();
+ db.import_header(header, cache, NewBlockState::Best, Vec::new()).unwrap();
hash
}
fn insert_final_block Header>(
db: &LightStorage,
- authorities: Option>>,
+ cache: HashMap>,
header: F,
) -> Hash {
let header = header();
let hash = header.hash();
- db.import_header(header, authorities, NewBlockState::Final, Vec::new()).unwrap();
+ db.import_header(header, cache, NewBlockState::Final, Vec::new()).unwrap();
hash
}
fn insert_non_best_block Header>(
db: &LightStorage,
- authorities: Option>>,
+ cache: HashMap>,
header: F,
) -> Hash {
let header = header();
let hash = header.hash();
- db.import_header(header, authorities, NewBlockState::Normal, Vec::new()).unwrap();
+ db.import_header(header, cache, NewBlockState::Normal, Vec::new()).unwrap();
hash
}
#[test]
fn returns_known_header() {
let db = LightStorage::new_test();
- let known_hash = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let known_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap();
let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap();
assert_eq!(header_by_hash, header_by_number);
@@ -616,12 +619,12 @@ pub(crate) mod tests {
#[test]
fn returns_info() {
let db = LightStorage::new_test();
- let genesis_hash = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let info = db.info().unwrap();
assert_eq!(info.best_hash, genesis_hash);
assert_eq!(info.best_number, 0);
assert_eq!(info.genesis_hash, genesis_hash);
- let best_hash = insert_block(&db, None, || default_header(&genesis_hash, 1));
+ let best_hash = insert_block(&db, HashMap::new(), || default_header(&genesis_hash, 1));
let info = db.info().unwrap();
assert_eq!(info.best_hash, best_hash);
assert_eq!(info.best_number, 1);
@@ -631,7 +634,7 @@ pub(crate) mod tests {
#[test]
fn returns_block_status() {
let db = LightStorage::new_test();
- let genesis_hash = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(), BlockStatus::Unknown);
@@ -641,7 +644,7 @@ pub(crate) mod tests {
#[test]
fn returns_block_hash() {
let db = LightStorage::new_test();
- let genesis_hash = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.hash(0).unwrap(), Some(genesis_hash));
assert_eq!(db.hash(1).unwrap(), None);
}
@@ -650,11 +653,11 @@ pub(crate) mod tests {
fn import_header_works() {
let db = LightStorage::new_test();
- let genesis_hash = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.db.iter(columns::HEADER).count(), 1);
assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), 2);
- let _ = insert_block(&db, None, || default_header(&genesis_hash, 1));
+ let _ = insert_block(&db, HashMap::new(), || default_header(&genesis_hash, 1));
assert_eq!(db.db.iter(columns::HEADER).count(), 2);
assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), 4);
}
@@ -665,25 +668,25 @@ pub(crate) mod tests {
let db = LightStorage::new_test();
// insert genesis block header (never pruned)
- let mut prev_hash = insert_final_block(&db, None, || header_producer(&Default::default(), 0));
+ let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0));
// insert SIZE blocks && ensure that nothing is pruned
for number in 0..cht::SIZE {
- prev_hash = insert_block(&db, None, || header_producer(&prev_hash, 1 + number));
+ prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number));
}
assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE) as usize);
assert_eq!(db.db.iter(columns::CHT).count(), 0);
// insert next SIZE blocks && ensure that nothing is pruned
for number in 0..cht::SIZE {
- prev_hash = insert_block(&db, None, || header_producer(&prev_hash, 1 + cht::SIZE + number));
+ prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + cht::SIZE + number));
}
assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + cht::SIZE) as usize);
assert_eq!(db.db.iter(columns::CHT).count(), 0);
// insert block #{2 * cht::SIZE + 1} && check that new CHT is created + headers of this CHT are pruned
// nothing is yet finalized, so nothing is pruned.
- prev_hash = insert_block(&db, None, || header_producer(&prev_hash, 1 + cht::SIZE + cht::SIZE));
+ prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + cht::SIZE + cht::SIZE));
assert_eq!(db.db.iter(columns::HEADER).count(), (2 + cht::SIZE + cht::SIZE) as usize);
assert_eq!(db.db.iter(columns::CHT).count(), 0);
@@ -732,9 +735,9 @@ pub(crate) mod tests {
let db = LightStorage::new_test();
// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
- let mut prev_hash = insert_final_block(&db, None, || header_with_changes_trie(&Default::default(), 0));
+ let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_with_changes_trie(&Default::default(), 0));
for i in 1..1 + cht::SIZE + cht::SIZE + 1 {
- prev_hash = insert_block(&db, None, || header_with_changes_trie(&prev_hash, i as u64));
+ prev_hash = insert_block(&db, HashMap::new(), || header_with_changes_trie(&prev_hash, i as u64));
db.finalize_header(BlockId::Hash(prev_hash)).unwrap();
}
@@ -754,16 +757,16 @@ pub(crate) mod tests {
#[test]
fn tree_route_works() {
let db = LightStorage::new_test();
- let block0 = insert_block(&db, None, || default_header(&Default::default(), 0));
+ let block0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
// fork from genesis: 3 prong.
- let a1 = insert_block(&db, None, || default_header(&block0, 1));
- let a2 = insert_block(&db, None, || default_header(&a1, 2));
- let a3 = insert_block(&db, None, || default_header(&a2, 3));
+ let a1 = insert_block(&db, HashMap::new(), || default_header(&block0, 1));
+ let a2 = insert_block(&db, HashMap::new(), || default_header(&a1, 2));
+ let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3));
// fork from genesis: 2 prong.
- let b1 = insert_block(&db, None, || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
- let b2 = insert_block(&db, None, || default_header(&b1, 2));
+ let b1 = insert_block(&db, HashMap::new(), || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
+ let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2));
{
let tree_route = ::client::blockchain::tree_route(
@@ -815,44 +818,64 @@ pub(crate) mod tests {
}
#[test]
- fn authorites_are_cached() {
+ fn authorities_are_cached() {
let db = LightStorage::new_test();
fn run_checks(db: &LightStorage, max: u64, checks: &[(u64, Option>>)]) {
for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) {
- let actual = db.cache().authorities_at(BlockId::Number(*at));
+ let actual = get_authorities(db.cache(), BlockId::Number(*at));
assert_eq!(*expected, actual);
}
}
+ fn same_authorities() -> HashMap