Commit 5f1e44be authored by Arkadiy Paronyan's avatar Arkadiy Paronyan Committed by Gav Wood
Browse files

Serialisable genesis config (#229)

* Genesis serialization

* Custom type for AuthorityId

* Merge w master

* Fixed a few minor issues

* Fixed unmerged file

* Renamed tag

* Deferred genesis loading

* Upated wasm runtime

* Minor issues
parent b715f329
......@@ -190,7 +190,7 @@ mod tests {
use client::LocalCallExecutor;
use client::in_mem::Backend as InMemory;
use substrate_executor::NativeExecutionDispatch;
use runtime::{GenesisConfig, ConsensusConfig, SessionConfig, BuildStorage};
use runtime::{GenesisConfig, ConsensusConfig, SessionConfig};
fn validators() -> Vec<AccountId> {
vec![
......@@ -201,8 +201,8 @@ mod tests {
fn session_keys() -> Vec<SessionKey> {
vec![
Keyring::One.to_raw_public(),
Keyring::Two.to_raw_public(),
Keyring::One.to_raw_public().into(),
Keyring::Two.to_raw_public().into(),
]
}
......@@ -225,7 +225,7 @@ mod tests {
timestamp: Some(Default::default()),
};
::client::new_in_mem(LocalDispatch::new(), genesis_config.build_storage()).unwrap()
::client::new_in_mem(LocalDispatch::new(), genesis_config).unwrap()
}
#[test]
......
......@@ -15,7 +15,6 @@ time = "0.1"
slog = "^2"
ansi_term = "0.10"
lazy_static = "1.0"
hex-literal = "0.1"
triehash = "0.1"
ed25519 = { path = "../../substrate/ed25519" }
app_dirs = "1.2"
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -16,6 +16,9 @@
//! Predefined chains.
use service;
use std::path::PathBuf;
/// The chain specification (this should eventually be replaced by a more general JSON-based chain
/// specification).
#[derive(Clone, Debug)]
......@@ -32,6 +35,19 @@ pub enum ChainSpec {
Custom(String),
}
/// Get a chain config from a spec setting.
impl ChainSpec {
pub(crate) fn load(self) -> Result<service::ChainSpec, String> {
Ok(match self {
ChainSpec::PoC1Testnet => service::ChainSpec::poc_1_testnet_config()?,
ChainSpec::Development => service::ChainSpec::development_config(),
ChainSpec::LocalTestnet => service::ChainSpec::local_testnet_config(),
ChainSpec::PoC2Testnet => service::ChainSpec::poc_2_testnet_config(),
ChainSpec::Custom(f) => service::ChainSpec::from_json_file(PathBuf::from(f))?,
})
}
}
impl<'a> From<&'a str> for ChainSpec {
fn from(s: &'a str) -> Self {
match s {
......@@ -55,19 +71,3 @@ impl From<ChainSpec> for String {
}
}
}
impl ::std::fmt::Display for ChainSpec {
fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
if let ChainSpec::Custom(n) = self {
write!(f, "Custom ({})", n)
} else {
write!(f, "{}", match *self {
ChainSpec::Development => "Development",
ChainSpec::LocalTestnet => "Local Testnet",
ChainSpec::PoC1Testnet => "PoC-1 Testnet",
ChainSpec::PoC2Testnet => "PoC-2 Testnet",
_ => unreachable!(),
})
}
}
}
......@@ -45,10 +45,6 @@ args:
long: dev
help: Run in development mode; implies --chain=dev --validator --key Alice
takes_value: false
- build-genesis:
long: build-genesis
help: Build a genesis.json file, outputing to stdout
takes_value: false
- port:
long: port
value_name: PORT
......@@ -96,3 +92,15 @@ args:
help: The URL of the telemetry server. Implies --telemetry
takes_value: true
subcommands:
- build-spec:
about: Build a spec.json file, outputing to stdout
args:
- raw:
long: raw
help: Force raw genesis storage output.
takes_value: false
- chain:
long: chain
value_name: CHAIN_SPEC
help: Specify the chain specification (one of dev, local or poc-2)
takes_value: true
......@@ -58,26 +58,17 @@ extern crate clap;
extern crate error_chain;
#[macro_use]
extern crate log;
#[macro_use]
extern crate hex_literal;
pub mod error;
mod informant;
mod chain_spec;
mod preset_config;
pub use chain_spec::ChainSpec;
pub use preset_config::PresetConfig;
use std::io;
use std::fs::File;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use substrate_primitives::hexdisplay::HexDisplay;
use substrate_primitives::storage::{StorageData, StorageKey};
use substrate_telemetry::{init_telemetry, TelemetryConfig};
use runtime_primitives::StorageMap;
use polkadot_primitives::Block;
use futures::sync::mpsc;
......@@ -106,10 +97,13 @@ impl substrate_rpc::system::SystemApi for SystemConfiguration {
}
}
fn read_storage_json(filename: &str) -> Option<StorageMap> {
let file = File::open(PathBuf::from(filename)).ok()?;
let h: HashMap<StorageKey, StorageData> = ::serde_json::from_reader(&file).ok()?;
Some(h.into_iter().map(|(k, v)| (k.0, v.0)).collect())
fn load_spec(matches: &clap::ArgMatches) -> Result<service::ChainSpec, String> {
let chain_spec = matches.value_of("chain")
.map(ChainSpec::from)
.unwrap_or_else(|| if matches.is_present("dev") { ChainSpec::Development } else { ChainSpec::PoC2Testnet });
let spec = chain_spec.load()?;
info!("Chain specification: {}", spec.name());
Ok(spec)
}
/// Parse command line arguments and start the node.
......@@ -124,8 +118,6 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let core = reactor::Core::new().expect("tokio::Core could not be created");
let yaml = load_yaml!("./cli.yml");
let matches = match clap::App::from_yaml(yaml).version(&(crate_version!().to_owned() + "\n")[..]).get_matches_from_safe(args) {
Ok(m) => m,
......@@ -146,20 +138,22 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
info!(" version {}", crate_version!());
info!(" by Parity Technologies, 2017, 2018");
let mut config = service::Configuration::default();
if let Some(matches) = matches.subcommand_matches("build-spec") {
let spec = load_spec(&matches)?;
info!("Building chain spec");
let json = spec.to_json(matches.is_present("raw"))?;
print!("{}", json);
return Ok(())
}
let spec = load_spec(&matches)?;
let mut config = service::Configuration::default_with_spec(spec);
if let Some(name) = matches.value_of("name") {
config.name = name.into();
info!("Node name: {}", config.name);
}
let chain_spec = matches.value_of("chain")
.map(ChainSpec::from)
.unwrap_or_else(|| if matches.is_present("dev") { ChainSpec::Development } else { ChainSpec::PoC2Testnet });
info!("Chain specification: {}", chain_spec);
config.chain_name = chain_spec.clone().into();
let base_path = matches.value_of("base-path")
.map(|x| Path::new(x).to_owned())
.unwrap_or_else(default_base_path);
......@@ -171,6 +165,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
.into();
config.database_path = db_path(&base_path).to_string_lossy().into();
config.pruning = match matches.value_of("pruning") {
Some("archive") => PruningMode::ArchiveAll,
None => PruningMode::keep_blocks(256),
......@@ -178,25 +173,6 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
.map_err(|_| error::ErrorKind::Input("Invalid pruning mode specified".to_owned()))?),
};
let (mut genesis_storage, boot_nodes) = PresetConfig::from_spec(chain_spec)
.map(PresetConfig::deconstruct)
.unwrap_or_else(|f| (Box::new(move ||
read_storage_json(&f)
.map(|s| { info!("{} storage items read from {}", s.len(), f); s })
.unwrap_or_else(|| panic!("Bad genesis state file: {}", f))
), vec![]));
if matches.is_present("build-genesis") {
info!("Building genesis");
for (i, (k, v)) in genesis_storage().iter().enumerate() {
print!("{}\n\"0x{}\": \"0x{}\"", if i > 0 {','} else {'{'}, HexDisplay::from(k), HexDisplay::from(v));
}
println!("\n}}");
return Ok(())
}
config.genesis_storage = genesis_storage;
let role =
if matches.is_present("collator") {
info!("Starting collator");
......@@ -214,10 +190,9 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
config.roles = role;
{
config.network.boot_nodes = matches
config.network.boot_nodes.extend(matches
.values_of("bootnodes")
.map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect());
config.network.boot_nodes.extend(boot_nodes);
.map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect::<Vec<_>>()));
config.network.config_path = Some(network_path(&base_path).to_string_lossy().into());
config.network.net_config_path = config.network.config_path.clone();
......@@ -241,12 +216,12 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
}
let sys_conf = SystemConfiguration {
chain_name: config.chain_name.clone(),
chain_name: config.chain_spec.name().to_owned(),
};
let _guard = if matches.is_present("telemetry") || matches.value_of("telemetry-url").is_some() {
let name = config.name.clone();
let chain_name = config.chain_name.clone();
let chain_name = config.chain_spec.name().to_owned();
Some(init_telemetry(TelemetryConfig {
url: matches.value_of("telemetry-url").unwrap_or(DEFAULT_TELEMETRY_URL).into(),
on_connect: Box::new(move || {
......@@ -263,6 +238,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
None
};
let core = reactor::Core::new().expect("tokio::Core could not be created");
match role == service::Role::LIGHT {
true => run_until_exit(core, service::new_light(config)?, &matches, sys_conf),
false => run_until_exit(core, service::new_full(config)?, &matches, sys_conf),
......
......@@ -261,7 +261,7 @@ impl<C, N, P> bft::ProposerFactory<Block> for ProposerFactory<C, N, P>
let (group_info, local_duty) = make_group_info(
duty_roster,
authorities,
sign_with.public().0,
sign_with.public().into(),
)?;
let active_parachains = self.client.active_parachains(&checked_id)?;
......@@ -490,7 +490,7 @@ impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
let offset = offset.low_u64() as usize + round_number;
let proposer = authorities[offset % authorities.len()].clone();
trace!(target: "bft", "proposer for round {} is {}", round_number, Hash::from(proposer));
trace!(target: "bft", "proposer for round {} is {}", round_number, proposer);
proposer
}
......
......@@ -92,16 +92,16 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
sender: proposal.sender,
digest_signature: ed25519::LocalizedSignature {
signature: proposal.digest_signature,
signer: ed25519::Public(proposal.sender),
signer: proposal.sender.into(),
},
full_signature: ed25519::LocalizedSignature {
signature: proposal.full_signature,
signer: ed25519::Public(proposal.sender),
signer: proposal.sender.into(),
}
};
bft::check_proposal(authorities, &msg.parent_hash, &proposal)?;
trace!(target: "bft", "importing proposal message for round {} from {}", proposal.round_number, Hash::from(proposal.sender));
trace!(target: "bft", "importing proposal message for round {} from {}", proposal.round_number, proposal.sender);
proposal
}),
net::generic_message::SignedConsensusMessage::Vote(vote) => bft::generic::LocalizedMessage::Vote({
......@@ -110,7 +110,7 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
sender: vote.sender,
signature: ed25519::LocalizedSignature {
signature: vote.signature,
signer: ed25519::Public(vote.sender),
signer: vote.sender.into(),
},
vote: match vote.vote {
net::generic_message::ConsensusVote::Prepare(r, h) => bft::generic::Vote::Prepare(r as usize, h),
......@@ -120,7 +120,7 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
};
bft::check_vote::<Block>(authorities, &msg.parent_hash, &vote)?;
trace!(target: "bft", "importing vote {:?} from {}", vote.vote, Hash::from(vote.sender));
trace!(target: "bft", "importing vote {:?} from {}", vote.vote, vote.sender);
vote
}),
}),
......
......@@ -62,17 +62,16 @@ impl table::Context for TableContext {
impl TableContext {
fn local_id(&self) -> AuthorityId {
self.key.public().0
self.key.public().into()
}
fn sign_statement(&self, statement: table::Statement) -> table::SignedStatement {
let signature = ::sign_table_statement(&statement, &self.key, &self.parent_hash).into();
let local_id = self.key.public().0;
table::SignedStatement {
statement,
signature,
sender: local_id,
sender: self.local_id(),
}
}
}
......@@ -468,10 +467,10 @@ mod tests {
let mut groups = HashMap::new();
let para_id = ParaId::from(1);
let local_id = Keyring::Alice.to_raw_public();
let local_id = Keyring::Alice.to_raw_public().into();
let local_key = Arc::new(Keyring::Alice.pair());
let validity_other = Keyring::Bob.to_raw_public();
let validity_other = Keyring::Bob.to_raw_public().into();
let validity_other_key = Keyring::Bob.pair();
let parent_hash = Default::default();
......@@ -518,10 +517,10 @@ mod tests {
let mut groups = HashMap::new();
let para_id = ParaId::from(1);
let local_id = Keyring::Alice.to_raw_public();
let local_id = Keyring::Alice.to_raw_public().into();
let local_key = Arc::new(Keyring::Alice.pair());
let validity_other = Keyring::Bob.to_raw_public();
let validity_other = Keyring::Bob.to_raw_public().into();
let validity_other_key = Keyring::Bob.pair();
let parent_hash = Default::default();
......
......@@ -136,7 +136,7 @@ pub type Timestamp = timestamp::Module<Concrete>;
pub struct SessionKeyConversion;
impl Convert<AccountId, SessionKey> for SessionKeyConversion {
fn convert(a: AccountId) -> SessionKey {
a.0
a.0.into()
}
}
......
......@@ -181,10 +181,14 @@ impl<T: Trait> Executable for Module<T> {
/// Parachains module genesis configuration.
#[cfg(any(feature = "std", test))]
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(deny_unknown_fields)]
pub struct GenesisConfig<T: Trait> {
/// The initial parachains, mapped to code.
pub parachains: Vec<(Id, Vec<u8>)>,
/// Phantom data.
#[serde(skip)]
pub phantom: PhantomData<T>,
}
......@@ -201,7 +205,7 @@ impl<T: Trait> Default for GenesisConfig<T> {
#[cfg(any(feature = "std", test))]
impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
{
fn build_storage(mut self) -> runtime_io::TestExternalities {
fn build_storage(mut self) -> ::std::result::Result<runtime_io::TestExternalities, String> {
use std::collections::HashMap;
use runtime_io::twox_128;
use codec::Slicable;
......@@ -220,7 +224,7 @@ impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
map.insert(key, code.encode());
}
map.into()
Ok(map.into())
}
}
......@@ -269,20 +273,20 @@ mod tests {
type Parachains = Module<Test>;
fn new_test_ext(parachains: Vec<(Id, Vec<u8>)>) -> runtime_io::TestExternalities {
let mut t = system::GenesisConfig::<Test>::default().build_storage();
let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap();
t.extend(consensus::GenesisConfig::<Test>{
code: vec![],
authorities: vec![1, 2, 3],
}.build_storage());
}.build_storage().unwrap());
t.extend(session::GenesisConfig::<Test>{
session_length: 1000,
validators: vec![1, 2, 3, 4, 5, 6, 7, 8],
broken_percent_late: 100,
}.build_storage());
}.build_storage().unwrap());
t.extend(GenesisConfig::<Test>{
parachains: parachains,
phantom: PhantomData,
}.build_storage());
}.build_storage().unwrap());
t
}
......
......@@ -14,6 +14,10 @@ slog = "^2"
clap = "2.27"
tokio-core = "0.1.12"
exit-future = "0.1"
serde = "1.0"
serde_json = "1.0"
serde_derive = "1.0"
hex-literal = "0.1"
ed25519 = { path = "../../substrate/ed25519" }
polkadot-primitives = { path = "../primitives" }
polkadot-runtime = { path = "../runtime" }
......
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -16,71 +16,127 @@
//! Polkadot chain configurations.
use std::collections::HashMap;
use ed25519;
use serde_json;
use substrate_primitives::{AuthorityId, storage::{StorageKey, StorageData}};
use runtime_primitives::{MakeStorage, BuildStorage, StorageMap};
use std::collections::HashMap;
use std::fs::File;
use std::path::PathBuf;
use primitives::{AuthorityId, storage::{StorageKey, StorageData}};
use runtime_primitives::{BuildStorage, StorageMap};
use polkadot_runtime::{GenesisConfig, ConsensusConfig, CouncilConfig, DemocracyConfig,
SessionConfig, StakingConfig, TimestampConfig};
use chain_spec::ChainSpec;
use serde_json as json;
enum Config {
Local(GenesisConfig),
Raw(&'static [u8]),
enum GenesisSource {
File(PathBuf),
Embedded(&'static [u8]),
Factory(fn() -> Genesis),
}
/// A configuration of a chain. Can be used to build a genesis block.
pub struct PresetConfig {
genesis_config: Config,
pub(crate) boot_nodes: Vec<String>,
impl GenesisSource {
fn resolve(&self) -> Result<Genesis, String> {
#[derive(Serialize, Deserialize)]
struct GenesisContainer {
genesis: Genesis,
}
match *self {
GenesisSource::File(ref path) => {
let file = File::open(path).map_err(|e| format!("Error opening spec file: {}", e))?;
let genesis: GenesisContainer = json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(genesis.genesis)
},
GenesisSource::Embedded(buf) => {
let genesis: GenesisContainer = json::from_reader(buf).map_err(|e| format!("Error parsing embedded file: {}", e))?;
Ok(genesis.genesis)
},
GenesisSource::Factory(f) => Ok(f()),
}
}
}
impl BuildStorage for Config {
fn build_storage(self) -> StorageMap {
match self {
Config::Local(gc) => gc.build_storage(),
Config::Raw(json) => {
let h: HashMap<StorageKey, StorageData> = serde_json::from_slice(json).expect("Data is from an internal source and is guaranteed to be of the correct format");
h.into_iter().map(|(k, v)| (k.0, v.0)).collect()
}
impl<'a> BuildStorage for &'a ChainSpec {
fn build_storage(self) -> Result<StorageMap, String> {
match self.genesis.resolve()? {
Genesis::Runtime(gc) => gc.build_storage(),
Genesis::Raw(map) => Ok(map.into_iter().map(|(k, v)| (k.0, v.0)).collect()),
}
}
}
impl PresetConfig {
/// Get a chain config from a spec, if it's predefined.
pub fn from_spec(chain_spec: ChainSpec) -> Result<Self, String> {
Ok(match chain_spec {
ChainSpec::PoC1Testnet => Self::poc_1_testnet_config(),
ChainSpec::Development => Self::development_config(),
ChainSpec::LocalTestnet => Self::local_testnet_config(),
ChainSpec::PoC2Testnet => Self::poc_2_testnet_config(),
ChainSpec::Custom(f) => return Err(f),
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(deny_unknown_fields)]
enum Genesis {
Runtime(GenesisConfig),
Raw(HashMap<StorageKey, StorageData>),
}
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ChainSpecFile {
pub name: String,
pub boot_nodes: Vec<String>,
}
/// A configuration of a chain. Can be used to build a genesis block.
pub struct ChainSpec {
spec: ChainSpecFile,
genesis: GenesisSource,
}
impl ChainSpec {
pub fn boot_nodes(&self) -> &[String] {
&self.spec.boot_nodes
}
pub fn name(&self) -> &str {
&self.spec.name
}
/// Parse json content into a `ChainSpec`
pub fn from_embedded(json: &'static [u8]) -> Result<Self, String> {
let spec = json::from_slice(json).map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(ChainSpec {
spec,
genesis: GenesisSource::Embedded(json),
})
}
/// Provide the boot nodes and a storage-builder function.
// TODO: Change return type to FnOnce as soon as Box<FnOnce> is callable or BoxFn is stablised.
pub fn deconstruct(self) -> (MakeStorage, Vec<String>) {
let mut gc = Some(self.genesis_config);
let f = move || gc.take().map(BuildStorage::build_storage).unwrap_or_default();
(Box::new(f), self.boot_nodes)
/// Parse json file into a `ChainSpec`
pub fn from_json_file(path: PathBuf) -> Result<Self, String> {
let file = File::open(&path).map_err(|e| format!("Error opening spec file: {}", e))?;
let spec = json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?;
Ok(ChainSpec {
spec,
genesis: GenesisSource::File(path),
})
}
/// PoC-1 testnet config.
fn poc_1_testnet_config() -> Self {
let genesis_config = Config::Raw(include_bytes!("../poc-1.json"));
let boot_nodes = vec![
"enode://a93a29fa68d965452bf0ff8c1910f5992fe2273a72a1ee8d3a3482f68512a61974211ba32bb33f051ceb1530b8ba3527fc36224ba6b9910329025e6d9153cf50@104.211.54.233:30333".into(),
"enode://051b18f63a316c4c5fef4631f8c550ae0adba179153588406fac3e5bbbbf534ebeda1bf475dceda27a531f6cdef3846ab6a010a269aa643a1fec7bff51af66bd@104.211.48.51:30333".into(),
"enode://c831ec9011d2c02d2c4620fc88db6d897a40d2f88fd75f47b9e4cf3b243999acb6f01b7b7343474650b34eeb1363041a422a91f1fc3850e43482983ee15aa582@104.211.48.247:30333".into(),
];
PresetConfig { genesis_config, boot_nodes }
/// Dump to json string.
pub fn to_json(self, raw: bool) -> Result<String, String> {
let genesis = match (raw, self.genesis.resolve()?) {
(true, Genesis::Runtime(g)) => {
let storage = g.build_storage()?.into_iter()
.map(|(k, v)| (StorageKey(k), StorageData(v)))
.collect();
Genesis::Raw(storage)
},