Serialisable genesis config (#229)

* Genesis serialization

* Custom type for AuthorityId

* Merge w master

* Fixed a few minor issues

* Fixed unmerged file

* Renamed tag

* Deferred genesis loading

* Upated wasm runtime

* Minor issues

api/src/full.rs

@@ -190,7 +190,7 @@ mod tests {
 	use client::LocalCallExecutor;
 	use client::in_mem::Backend as InMemory;
 	use substrate_executor::NativeExecutionDispatch;
-	use runtime::{GenesisConfig, ConsensusConfig, SessionConfig, BuildStorage};
+	use runtime::{GenesisConfig, ConsensusConfig, SessionConfig};
 
 	fn validators() -> Vec<AccountId> {
 		vec![
@@ -201,8 +201,8 @@ mod tests {
 
 	fn session_keys() -> Vec<SessionKey> {
 		vec![
-			Keyring::One.to_raw_public(),
-			Keyring::Two.to_raw_public(),
+			Keyring::One.to_raw_public().into(),
+			Keyring::Two.to_raw_public().into(),
 		]
 	}
 
@@ -225,7 +225,7 @@ mod tests {
 			timestamp: Some(Default::default()),
 		};
 
-		::client::new_in_mem(LocalDispatch::new(), genesis_config.build_storage()).unwrap()
+		::client::new_in_mem(LocalDispatch::new(), genesis_config).unwrap()
 	}
 
 	#[test]

cli/Cargo.toml

@@ -15,7 +15,6 @@ time = "0.1"
 slog = "^2"
 ansi_term = "0.10"
 lazy_static = "1.0"
-hex-literal = "0.1"
 triehash = "0.1"
 ed25519 = { path = "../../substrate/ed25519" }
 app_dirs = "1.2"

cli/src/chain_spec.rs

@@ -16,6 +16,9 @@
 
 //! Predefined chains.
 
+use service;
+use std::path::PathBuf;
+
 /// The chain specification (this should eventually be replaced by a more general JSON-based chain
 /// specification).
 #[derive(Clone, Debug)]
@@ -32,6 +35,19 @@ pub enum ChainSpec {
 	Custom(String),
 }
 
+/// Get a chain config from a spec setting.
+impl ChainSpec {
+	pub(crate) fn load(self) -> Result<service::ChainSpec, String> {
+		Ok(match self {
+			ChainSpec::PoC1Testnet => service::ChainSpec::poc_1_testnet_config()?,
+			ChainSpec::Development => service::ChainSpec::development_config(),
+			ChainSpec::LocalTestnet => service::ChainSpec::local_testnet_config(),
+			ChainSpec::PoC2Testnet => service::ChainSpec::poc_2_testnet_config(),
+			ChainSpec::Custom(f) => service::ChainSpec::from_json_file(PathBuf::from(f))?,
+		})
+	}
+}
+
 impl<'a> From<&'a str> for ChainSpec {
 	fn from(s: &'a str) -> Self {
 		match s {
@@ -55,19 +71,3 @@ impl From<ChainSpec> for String {
 		}
 	}
 }
-
-impl ::std::fmt::Display for ChainSpec {
-	fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-		if let ChainSpec::Custom(n) = self {
-			write!(f, "Custom ({})", n)
-		} else {
-			write!(f, "{}", match *self {
-				ChainSpec::Development => "Development",
-				ChainSpec::LocalTestnet => "Local Testnet",
-				ChainSpec::PoC1Testnet => "PoC-1 Testnet",
-				ChainSpec::PoC2Testnet => "PoC-2 Testnet",
-				_ => unreachable!(),
-			})
-		}
-	}
-}

cli/src/cli.yml

@@ -45,10 +45,6 @@ args:
       long: dev
       help: Run in development mode; implies --chain=dev --validator --key Alice
       takes_value: false
-  - build-genesis:
-      long: build-genesis
-      help: Build a genesis.json file, outputing to stdout
-      takes_value: false
   - port:
       long: port
       value_name: PORT
@@ -96,3 +92,15 @@ args:
       help: The URL of the telemetry server. Implies --telemetry
       takes_value: true
 subcommands:
+  - build-spec:
+      about: Build a spec.json file, outputing to stdout
+      args:
+        - raw:
+            long: raw
+            help: Force raw genesis storage output.
+            takes_value: false
+        - chain:
+            long: chain
+            value_name: CHAIN_SPEC
+            help: Specify the chain specification (one of dev, local or poc-2)
+            takes_value: true

cli/src/lib.rs

@@ -58,26 +58,17 @@ extern crate clap;
 extern crate error_chain;
 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate hex_literal;
 
 pub mod error;
 mod informant;
 mod chain_spec;
-mod preset_config;
 
 pub use chain_spec::ChainSpec;
-pub use preset_config::PresetConfig;
 
 use std::io;
-use std::fs::File;
 use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
-use std::collections::HashMap;
-use substrate_primitives::hexdisplay::HexDisplay;
-use substrate_primitives::storage::{StorageData, StorageKey};
 use substrate_telemetry::{init_telemetry, TelemetryConfig};
-use runtime_primitives::StorageMap;
 use polkadot_primitives::Block;
 
 use futures::sync::mpsc;
@@ -106,10 +97,13 @@ impl substrate_rpc::system::SystemApi for SystemConfiguration {
 	}
 }
 
-fn read_storage_json(filename: &str) -> Option<StorageMap> {
-	let file = File::open(PathBuf::from(filename)).ok()?;
-	let h: HashMap<StorageKey, StorageData> = ::serde_json::from_reader(&file).ok()?;
-	Some(h.into_iter().map(|(k, v)| (k.0, v.0)).collect())
+fn load_spec(matches: &clap::ArgMatches) -> Result<service::ChainSpec, String> {
+	let chain_spec = matches.value_of("chain")
+		.map(ChainSpec::from)
+		.unwrap_or_else(|| if matches.is_present("dev") { ChainSpec::Development } else { ChainSpec::PoC2Testnet });
+	let spec = chain_spec.load()?;
+	info!("Chain specification: {}", spec.name());
+	Ok(spec)
 }
 
 /// Parse command line arguments and start the node.
@@ -124,8 +118,6 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	I: IntoIterator<Item = T>,
 	T: Into<std::ffi::OsString> + Clone,
 {
-	let core = reactor::Core::new().expect("tokio::Core could not be created");
-
 	let yaml = load_yaml!("./cli.yml");
 	let matches = match clap::App::from_yaml(yaml).version(&(crate_version!().to_owned() + "\n")[..]).get_matches_from_safe(args) {
 		Ok(m) => m,
@@ -146,20 +138,22 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	info!("  version {}", crate_version!());
 	info!("  by Parity Technologies, 2017, 2018");
 
-	let mut config = service::Configuration::default();
+	if let Some(matches) = matches.subcommand_matches("build-spec") {
+		let spec = load_spec(&matches)?;
+		info!("Building chain spec");
+		let json = spec.to_json(matches.is_present("raw"))?;
+		print!("{}", json);
+		return Ok(())
+	}
+
+	let spec = load_spec(&matches)?;
+	let mut config = service::Configuration::default_with_spec(spec);
 
 	if let Some(name) = matches.value_of("name") {
 		config.name = name.into();
 		info!("Node name: {}", config.name);
 	}
 
-	let chain_spec = matches.value_of("chain")
-		.map(ChainSpec::from)
-		.unwrap_or_else(|| if matches.is_present("dev") { ChainSpec::Development } else { ChainSpec::PoC2Testnet });
-	info!("Chain specification: {}", chain_spec);
-
-	config.chain_name = chain_spec.clone().into();
-
 	let base_path = matches.value_of("base-path")
 		.map(|x| Path::new(x).to_owned())
 		.unwrap_or_else(default_base_path);
@@ -171,6 +165,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 		.into();
 
 	config.database_path = db_path(&base_path).to_string_lossy().into();
+
 	config.pruning = match matches.value_of("pruning") {
 		Some("archive") => PruningMode::ArchiveAll,
 		None => PruningMode::keep_blocks(256),
@@ -178,25 +173,6 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 			.map_err(|_| error::ErrorKind::Input("Invalid pruning mode specified".to_owned()))?),
 	};
 
-	let (mut genesis_storage, boot_nodes) = PresetConfig::from_spec(chain_spec)
-		.map(PresetConfig::deconstruct)
-		.unwrap_or_else(|f| (Box::new(move ||
-			read_storage_json(&f)
-				.map(|s| { info!("{} storage items read from {}", s.len(), f); s })
-				.unwrap_or_else(|| panic!("Bad genesis state file: {}", f))
-		), vec![]));
-
-	if matches.is_present("build-genesis") {
-		info!("Building genesis");
-		for (i, (k, v)) in genesis_storage().iter().enumerate() {
-			print!("{}\n\"0x{}\": \"0x{}\"", if i > 0 {','} else {'{'}, HexDisplay::from(k), HexDisplay::from(v));
-		}
-		println!("\n}}");
-		return Ok(())
-	}
-
-	config.genesis_storage = genesis_storage;
-
 	let role =
 		if matches.is_present("collator") {
 			info!("Starting collator");
@@ -214,10 +190,9 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 
 	config.roles = role;
 	{
-		config.network.boot_nodes = matches
+		config.network.boot_nodes.extend(matches
 			.values_of("bootnodes")
-			.map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect());
-		config.network.boot_nodes.extend(boot_nodes);
+			.map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect::<Vec<_>>()));
 		config.network.config_path = Some(network_path(&base_path).to_string_lossy().into());
 		config.network.net_config_path = config.network.config_path.clone();
 
@@ -241,12 +216,12 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	}
 
 	let sys_conf = SystemConfiguration {
-		chain_name: config.chain_name.clone(),
+		chain_name: config.chain_spec.name().to_owned(),
 	};
 
 	let _guard = if matches.is_present("telemetry") || matches.value_of("telemetry-url").is_some() {
 		let name = config.name.clone();
-		let chain_name = config.chain_name.clone();
+		let chain_name = config.chain_spec.name().to_owned();
 		Some(init_telemetry(TelemetryConfig {
 			url: matches.value_of("telemetry-url").unwrap_or(DEFAULT_TELEMETRY_URL).into(),
 			on_connect: Box::new(move || {
@@ -263,6 +238,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 		None
 	};
 
+	let core = reactor::Core::new().expect("tokio::Core could not be created");
 	match role == service::Role::LIGHT {
 		true => run_until_exit(core, service::new_light(config)?, &matches, sys_conf),
 		false => run_until_exit(core, service::new_full(config)?, &matches, sys_conf),

consensus/src/lib.rs

@@ -261,7 +261,7 @@ impl<C, N, P> bft::ProposerFactory<Block> for ProposerFactory<C, N, P>
 		let (group_info, local_duty) = make_group_info(
 			duty_roster,
 			authorities,
-			sign_with.public().0,
+			sign_with.public().into(),
 		)?;
 
 		let active_parachains = self.client.active_parachains(&checked_id)?;
@@ -490,7 +490,7 @@ impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
 		let offset = offset.low_u64() as usize + round_number;
 
 		let proposer = authorities[offset % authorities.len()].clone();
-		trace!(target: "bft", "proposer for round {} is {}", round_number, Hash::from(proposer));
+		trace!(target: "bft", "proposer for round {} is {}", round_number, proposer);
 
 		proposer
 	}

consensus/src/service.rs

@@ -92,16 +92,16 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
 					sender: proposal.sender,
 					digest_signature: ed25519::LocalizedSignature {
 						signature: proposal.digest_signature,
-						signer: ed25519::Public(proposal.sender),
+						signer: proposal.sender.into(),
 					},
 					full_signature: ed25519::LocalizedSignature {
 						signature: proposal.full_signature,
-						signer: ed25519::Public(proposal.sender),
+						signer: proposal.sender.into(),
 					}
 				};
 				bft::check_proposal(authorities, &msg.parent_hash, &proposal)?;
 
-				trace!(target: "bft", "importing proposal message for round {} from {}", proposal.round_number, Hash::from(proposal.sender));
+				trace!(target: "bft", "importing proposal message for round {} from {}", proposal.round_number, proposal.sender);
 				proposal
 			}),
 			net::generic_message::SignedConsensusMessage::Vote(vote) => bft::generic::LocalizedMessage::Vote({
@@ -110,7 +110,7 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
 					sender: vote.sender,
 					signature: ed25519::LocalizedSignature {
 						signature: vote.signature,
-						signer: ed25519::Public(vote.sender),
+						signer: vote.sender.into(),
 					},
 					vote: match vote.vote {
 						net::generic_message::ConsensusVote::Prepare(r, h) => bft::generic::Vote::Prepare(r as usize, h),
@@ -120,7 +120,7 @@ fn process_message(msg: net::LocalizedBftMessage<Block>, local_id: &AuthorityId,
 				};
 				bft::check_vote::<Block>(authorities, &msg.parent_hash, &vote)?;
 
-				trace!(target: "bft", "importing vote {:?} from {}", vote.vote, Hash::from(vote.sender));
+				trace!(target: "bft", "importing vote {:?} from {}", vote.vote, vote.sender);
 				vote
 			}),
 		}),

consensus/src/shared_table/mod.rs

@@ -62,17 +62,16 @@ impl table::Context for TableContext {
 
 impl TableContext {
 	fn local_id(&self) -> AuthorityId {
-		self.key.public().0
+		self.key.public().into()
 	}
 
 	fn sign_statement(&self, statement: table::Statement) -> table::SignedStatement {
 		let signature = ::sign_table_statement(&statement, &self.key, &self.parent_hash).into();
-		let local_id = self.key.public().0;
 
 		table::SignedStatement {
 			statement,
 			signature,
-			sender: local_id,
+			sender: self.local_id(),
 		}
 	}
 }
@@ -468,10 +467,10 @@ mod tests {
 		let mut groups = HashMap::new();
 
 		let para_id = ParaId::from(1);
-		let local_id = Keyring::Alice.to_raw_public();
+		let local_id = Keyring::Alice.to_raw_public().into();
 		let local_key = Arc::new(Keyring::Alice.pair());
 
-		let validity_other = Keyring::Bob.to_raw_public();
+		let validity_other = Keyring::Bob.to_raw_public().into();
 		let validity_other_key = Keyring::Bob.pair();
 		let parent_hash = Default::default();
 
@@ -518,10 +517,10 @@ mod tests {
 		let mut groups = HashMap::new();
 
 		let para_id = ParaId::from(1);
-		let local_id = Keyring::Alice.to_raw_public();
+		let local_id = Keyring::Alice.to_raw_public().into();
 		let local_key = Arc::new(Keyring::Alice.pair());
 
-		let validity_other = Keyring::Bob.to_raw_public();
+		let validity_other = Keyring::Bob.to_raw_public().into();
 		let validity_other_key = Keyring::Bob.pair();
 		let parent_hash = Default::default();
 

runtime/src/lib.rs

@@ -136,7 +136,7 @@ pub type Timestamp = timestamp::Module<Concrete>;
 pub struct SessionKeyConversion;
 impl Convert<AccountId, SessionKey> for SessionKeyConversion {
 	fn convert(a: AccountId) -> SessionKey {
-		a.0
+		a.0.into()
 	}
 }
 

runtime/src/parachains.rs

@@ -181,10 +181,14 @@ impl<T: Trait> Executable for Module<T> {
 
 /// Parachains module genesis configuration.
 #[cfg(any(feature = "std", test))]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
 pub struct GenesisConfig<T: Trait> {
 	/// The initial parachains, mapped to code.
 	pub parachains: Vec<(Id, Vec<u8>)>,
 	/// Phantom data.
+	#[serde(skip)]
 	pub phantom: PhantomData<T>,
 }
 
@@ -201,7 +205,7 @@ impl<T: Trait> Default for GenesisConfig<T> {
 #[cfg(any(feature = "std", test))]
 impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
 {
-	fn build_storage(mut self) -> runtime_io::TestExternalities {
+	fn build_storage(mut self) -> ::std::result::Result<runtime_io::TestExternalities, String> {
 		use std::collections::HashMap;
 		use runtime_io::twox_128;
 		use codec::Slicable;
@@ -220,7 +224,7 @@ impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
 			map.insert(key, code.encode());
 		}
 
-		map.into()
+		Ok(map.into())
 	}
 }
 
@@ -269,20 +273,20 @@ mod tests {
 	type Parachains = Module<Test>;
 
 	fn new_test_ext(parachains: Vec<(Id, Vec<u8>)>) -> runtime_io::TestExternalities {
-		let mut t = system::GenesisConfig::<Test>::default().build_storage();
+		let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap();
 		t.extend(consensus::GenesisConfig::<Test>{
 			code: vec![],
 			authorities: vec![1, 2, 3],
-		}.build_storage());
+		}.build_storage().unwrap());
 		t.extend(session::GenesisConfig::<Test>{
 			session_length: 1000,
 			validators: vec![1, 2, 3, 4, 5, 6, 7, 8],
 			broken_percent_late: 100,
-		}.build_storage());