Bump to latest Substrate (#898)

* Flag to force kusama runtime

* Chainspecs for kusama

* Polkadot config for westend

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* network/src/legacy/gossip: Wrap GossipEngine in Arc Mutex & lock it on use

`GossipEngine` in itself has no need to be Send and Sync, given that it
does not rely on separately spawned background tasks anymore.
`RegisteredMessageValidator` needs to be `Send` and `Sync` due to the
inherited trait bounds from implementing `GossipService`. In addition
`RegisteredMessageValidator` derives `Clone`. Thereby `GossipEngine`
needs to be wrapped in an `Arc` and `Mutex` to keep the status quo.

* Needed fixes.

* Fixes

* Fixed build

* Fixed build w benchmarking CLI

* Fixed building tests

* Added --dev shortcut

Co-authored-by: arkpar <arkady.paronyan@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Max Inden <mail@max-inden.de>

availability-store/src/lib.rs

@@ -35,7 +35,7 @@ use polkadot_primitives::{
 use sp_runtime::traits::HashFor;
 use sp_blockchain::{Result as ClientResult};
 use client::{
-	BlockchainEvents, BlockBody,
+	BlockchainEvents, BlockBackend,
 };
 use sp_api::{ApiExt, ProvideRuntimeApi};
 use codec::{Encode, Decode};
@@ -178,7 +178,7 @@ impl Store {
 		keystore: KeyStorePtr,
 	) -> ClientResult<AvailabilityBlockImport<I, P>>
 	where
-		P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
+		P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBackend<Block> + Send + Sync + 'static,
 		P::Api: ParachainHost<Block>,
 		P::Api: ApiExt<Block, Error=sp_blockchain::Error>,
 		// Rust bug: https://github.com/rust-lang/rust/issues/24159

availability-store/src/worker.rs

@@ -24,7 +24,7 @@ use sp_blockchain::{Result as ClientResult};
 use sp_runtime::traits::{Header as HeaderT, Block as BlockT, HashFor, BlakeTwo256};
 use sp_api::{ApiExt, ProvideRuntimeApi};
 use client::{
-	BlockchainEvents, BlockBody,
+	BlockchainEvents, BlockBackend,
 	blockchain::ProvideCache,
 };
 use consensus_common::{
@@ -203,7 +203,7 @@ where
 /// Creates a task to prune entries in availability store upon block finalization.
 async fn prune_unneeded_availability<P, S>(client: Arc<P>, mut sender: S)
 where
-	P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBody<Block> + Send + Sync + 'static,
+	P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBackend<Block> + Send + Sync + 'static,
 	P::Api: ParachainHost<Block> + ApiExt<Block, Error=sp_blockchain::Error>,
 	S: Sink<WorkerMsg> + Clone + Send + Sync + Unpin,
 	// Rust bug: https://github.com/rust-lang/rust/issues/24159
@@ -646,7 +646,7 @@ impl<I, P> AvailabilityBlockImport<I, P> {
 		to_worker: mpsc::UnboundedSender<WorkerMsg>,
 	) -> Self
 	where
-		P: ProvideRuntimeApi<Block> + BlockBody<Block> + BlockchainEvents<Block> + Send + Sync + 'static,
+		P: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockchainEvents<Block> + Send + Sync + 'static,
 		P::Api: ParachainHost<Block>,
 		P::Api: ApiExt<Block, Error = sp_blockchain::Error>,
 		// Rust bug: https://github.com/rust-lang/rust/issues/24159

cli/src/chain_spec.rs

@@ -22,16 +22,22 @@ use service;
 /// specification).
 #[derive(Clone, Debug)]
 pub enum ChainSpec {
-	/// Whatever the current runtime is, with just Alice as an auth.
-	Development,
-	/// Whatever the current runtime is, with simple Alice/Bob auths.
-	LocalTestnet,
+	/// Whatever the current polkadot runtime is, with just Alice as an auth.
+	PolkadotDevelopment,
+	/// Whatever the current pokadot runtime is, with simple Alice/Bob auths.
+	PolkadotLocalTestnet,
 	/// The Kusama network.
 	Kusama,
+	/// Whatever the current kusama runtime is, with just Alice as an auth.
+	KusamaDevelopment,
 	/// The Westend network,
 	Westend,
-	/// Whatever the current runtime is with the "global testnet" defaults.
-	StagingTestnet,
+	/// Whatever the current polkadot runtime is with the "global testnet" defaults.
+	PolkadotStagingTestnet,
+	/// Whatever the current kusama runtime is with the "global testnet" defaults.
+	KusamaStagingTestnet,
+	/// Whatever the current kusama runtime is, with simple Alice/Bob auths.
+	KusamaLocalTestnet,
 }
 
 impl Default for ChainSpec {
@@ -42,23 +48,29 @@ impl Default for ChainSpec {
 
 /// Get a chain config from a spec setting.
 impl ChainSpec {
-	pub(crate) fn load(self) -> Result<service::ChainSpec, String> {
-		match self {
-			ChainSpec::Development => Ok(service::chain_spec::development_config()),
-			ChainSpec::LocalTestnet => Ok(service::chain_spec::local_testnet_config()),
-			ChainSpec::StagingTestnet => Ok(service::chain_spec::staging_testnet_config()),
-			ChainSpec::Westend => service::chain_spec::westend_config(),
-			ChainSpec::Kusama => service::chain_spec::kusama_config(),
-		}
+	pub(crate) fn load(self) -> Result<Box<dyn service::ChainSpec>, String> {
+		Ok(match self {
+			ChainSpec::PolkadotDevelopment => Box::new(service::chain_spec::polkadot_development_config()),
+			ChainSpec::PolkadotLocalTestnet => Box::new(service::chain_spec::polkadot_local_testnet_config()),
+			ChainSpec::PolkadotStagingTestnet => Box::new(service::chain_spec::polkadot_staging_testnet_config()),
+			ChainSpec::KusamaDevelopment =>Box::new(service::chain_spec::kusama_development_config()),
+			ChainSpec::KusamaLocalTestnet => Box::new(service::chain_spec::kusama_local_testnet_config()),
+			ChainSpec::KusamaStagingTestnet => Box::new(service::chain_spec::kusama_staging_testnet_config()),
+			ChainSpec::Westend => Box::new(service::chain_spec::westend_config()?),
+			ChainSpec::Kusama => Box::new(service::chain_spec::kusama_config()?),
+		})
 	}
 
 	pub(crate) fn from(s: &str) -> Option<Self> {
 		match s {
-			"dev" => Some(ChainSpec::Development),
-			"local" => Some(ChainSpec::LocalTestnet),
+			"polkadot-dev" | "dev" => Some(ChainSpec::PolkadotDevelopment),
+			"polkadot-local" => Some(ChainSpec::PolkadotLocalTestnet),
+			"polkadot-staging" => Some(ChainSpec::PolkadotStagingTestnet),
+			"kusama-dev" => Some(ChainSpec::KusamaDevelopment),
+			"kusama-local" => Some(ChainSpec::KusamaLocalTestnet),
+			"kusama-staging" => Some(ChainSpec::KusamaStagingTestnet),
 			"kusama" => Some(ChainSpec::Kusama),
 			"westend" => Some(ChainSpec::Westend),
-			"staging" => Some(ChainSpec::StagingTestnet),
 			"" => Some(ChainSpec::default()),
 			_ => None,
 		}
@@ -66,9 +78,11 @@ impl ChainSpec {
 }
 
 /// Load the `ChainSpec` for the given `id`.
-pub fn load_spec(id: &str) -> Result<Option<service::ChainSpec>, String> {
+/// `force_kusama` treats chain specs coming from a file as kusama specs.
+pub fn load_spec(id: &str, force_kusama: bool) -> Result<Box<dyn service::ChainSpec>, String> {
 	Ok(match ChainSpec::from(id) {
-		Some(spec) => Some(spec.load()?),
-		None => None,
+		Some(spec) => spec.load()?,
+		None if force_kusama => Box::new(service::KusamaChainSpec::from_json_file(std::path::PathBuf::from(id))?),
+		None => Box::new(service::PolkadotChainSpec::from_json_file(std::path::PathBuf::from(id))?),
 	})
 }

cli/src/cli.rs

@@ -17,7 +17,6 @@
 //! Polkadot CLI library.
 
 use structopt::StructOpt;
-pub use sc_cli::RunCmd;
 
 #[allow(missing_docs)]
 #[derive(Debug, StructOpt, Clone)]
@@ -45,6 +44,18 @@ pub struct ValidationWorkerCommand {
 	pub mem_id: String,
 }
 
+#[allow(missing_docs)]
+#[derive(Debug, StructOpt, Clone)]
+pub struct RunCmd {
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub base: sc_cli::RunCmd,
+
+	/// Force using Kusama native runtime.
+	#[structopt(long = "force-kusama")]
+	pub force_kusama: bool,
+}
+
 #[allow(missing_docs)]
 #[derive(Debug, StructOpt, Clone)]
 #[structopt(settings = &[

cli/src/command.rs

@@ -29,13 +29,18 @@ pub fn run(version: VersionInfo) -> sc_cli::Result<()> {
 
 	let mut config = service::Configuration::from_version(&version);
 	config.impl_name = "parity-polkadot";
+	let force_kusama = opt.run.force_kusama;
 
 	match opt.subcommand {
 		None => {
-			opt.run.init(&version)?;
-			opt.run.update_config(&mut config, load_spec, &version)?;
+			opt.run.base.init(&version)?;
+			opt.run.base.update_config(
+				&mut config,
+				|id| load_spec(id, force_kusama),
+				&version
+			)?;
 
-			let is_kusama = config.chain_spec.as_ref().map_or(false, |s| s.is_kusama());
+			let is_kusama = config.expect_chain_spec().is_kusama();
 
 			info!("{}", version.name);
 			info!("  version {}", config.full_version());
@@ -69,9 +74,13 @@ pub fn run(version: VersionInfo) -> sc_cli::Result<()> {
 		},
 		Some(Subcommand::Base(cmd)) => {
 			cmd.init(&version)?;
-			cmd.update_config(&mut config, load_spec, &version)?;
+			cmd.update_config(
+				&mut config,
+				|id| load_spec(id, force_kusama),
+				&version
+			)?;
 
-			let is_kusama = config.chain_spec.as_ref().map_or(false, |s| s.is_kusama());
+			let is_kusama = config.expect_chain_spec().is_kusama();
 
 			if is_kusama {
 				cmd.run(config, service::new_chain_ops::<
@@ -100,14 +109,12 @@ pub fn run(version: VersionInfo) -> sc_cli::Result<()> {
 		},
 		Some(Subcommand::Benchmark(cmd)) => {
 			cmd.init(&version)?;
-			cmd.update_config(&mut config, load_spec, &version)?;
-
-			let is_kusama = config.chain_spec.as_ref().map_or(false, |s| s.is_kusama());
-
+			cmd.update_config(&mut config, |id| load_spec(id, force_kusama), &version)?;
+			let is_kusama = config.expect_chain_spec().is_kusama();
 			if is_kusama {
-				cmd.run::<_, _, service::kusama_runtime::Block, service::KusamaExecutor>(config)
+				cmd.run::<service::kusama_runtime::Block, service::KusamaExecutor>(config)
 			} else {
-				cmd.run::<_, _, service::polkadot_runtime::Block, service::PolkadotExecutor>(config)
+				cmd.run::<service::polkadot_runtime::Block, service::PolkadotExecutor>(config)
 			}
 		},
 	}

network/src/legacy/gossip/mod.rs

@@ -71,7 +71,7 @@ use std::sync::Arc;
 
 use arrayvec::ArrayVec;
 use futures::prelude::*;
-use parking_lot::RwLock;
+use parking_lot::{Mutex, RwLock};
 
 use crate::legacy::{GossipMessageStream, GossipService};
 
@@ -289,21 +289,29 @@ pub fn register_validator<C: ChainContext + 'static>(
 	});
 
 	let gossip_side = validator.clone();
-	let gossip_engine = sc_network_gossip::GossipEngine::new(
+	let gossip_engine = Arc::new(Mutex::new(sc_network_gossip::GossipEngine::new(
 		service.clone(),
 		POLKADOT_ENGINE_ID,
 		POLKADOT_PROTOCOL_NAME,
 		gossip_side,
-	);
+	)));
 
+	// Spawn gossip engine.
+	//
 	// Ideally this would not be spawned as an orphaned task, but polled by
 	// `RegisteredMessageValidator` which in turn would be polled by a `ValidationNetwork`.
-	let spawn_res = executor.spawn_obj(futures::task::FutureObj::from(Box::new(gossip_engine.clone())));
+	{
+		let gossip_engine = gossip_engine.clone();
+		let fut = futures::future::poll_fn(move |cx| {
+			gossip_engine.lock().poll_unpin(cx)
+		});
+		let spawn_res = executor.spawn_obj(futures::task::FutureObj::from(Box::new(fut)));
 
 		// Note: we consider the chances of an error to spawn a background task almost null.
 		if spawn_res.is_err() {
 			log::error!(target: "polkadot-gossip", "Failed to spawn background task");
 		}
+	}
 
 	RegisteredMessageValidator {
 		inner: validator as _,
@@ -350,7 +358,7 @@ pub struct RegisteredMessageValidator {
 	// Note: this is always `Some` in real code and `None` in tests.
 	service: Option<Arc<NetworkService<Block, Hash>>>,
 	// Note: this is always `Some` in real code and `None` in tests.
-	gossip_engine: Option<sc_network_gossip::GossipEngine<Block>>,
+	gossip_engine: Option<Arc<Mutex<sc_network_gossip::GossipEngine<Block>>>>,
 }
 
 impl RegisteredMessageValidator {
@@ -398,7 +406,7 @@ impl RegisteredMessageValidator {
 
 	pub(crate) fn gossip_messages_for(&self, topic: Hash) -> GossipMessageStream {
 		let topic_stream = if let Some(gossip_engine) = self.gossip_engine.as_ref() {
-			gossip_engine.messages_for(topic)
+			gossip_engine.lock().messages_for(topic)
 		} else {
 			log::error!("Called gossip_messages_for on a test engine");
 			futures::channel::mpsc::unbounded().1
@@ -409,7 +417,7 @@ impl RegisteredMessageValidator {
 
 	pub(crate) fn gossip_message(&self, topic: Hash, message: GossipMessage) {
 		if let Some(gossip_engine) = self.gossip_engine.as_ref() {
-			gossip_engine.gossip_message(
+			gossip_engine.lock().gossip_message(
 				topic,
 				message.encode(),
 				false,
@@ -421,7 +429,7 @@ impl RegisteredMessageValidator {
 
 	pub(crate) fn send_message(&self, who: PeerId, message: GossipMessage) {
 		if let Some(gossip_engine) = self.gossip_engine.as_ref() {
-			gossip_engine.send_message(vec![who], message.encode());
+			gossip_engine.lock().send_message(vec![who], message.encode());
 		} else {
 			log::error!("Called send_message on a test engine");
 		}

network/test/src/lib.rs

@@ -31,6 +31,7 @@ use sc_client_api::{
 	BlockchainEvents, BlockImportNotification,
 	FinalityNotifications, ImportNotifications,
 	FinalityNotification,
+	client::BlockBackend,
 	backend::{TransactionFor, AuxStore, Backend, Finalizer},
 };
 use sc_block_builder::{BlockBuilder, BlockBuilderProvider};

parachain/src/wasm_executor/mod.rs

@@ -164,9 +164,11 @@ pub fn validate_candidate_internal<E: Externalities + 'static>(
 		Some(1024),
 		HostFunctions::host_functions(),
 		false,
+		8
 	);
 	let res = executor.call_in_wasm(
 		validation_code,
+		None,
 		"validate_block",
 		encoded_call_data,
 		&mut ext,
@@ -192,22 +194,6 @@ impl sp_externalities::Externalities for ValidationExternalities {
 		panic!("child_storage_hash: unsupported feature for parachain validation")
 	}
 
-	fn original_storage(&self, _: &[u8]) -> Option<Vec<u8>> {
-		panic!("original_sorage: unsupported feature for parachain validation")
-	}
-
-	fn original_child_storage(&self, _: ChildStorageKey, _: ChildInfo, _: &[u8]) -> Option<Vec<u8>> {
-		panic!("original_child_storage: unsupported feature for parachain validation")
-	}
-
-	fn original_storage_hash(&self, _: &[u8]) -> Option<Vec<u8>> {
-		panic!("original_storage_hash: unsupported feature for parachain validation")
-	}
-
-	fn original_child_storage_hash(&self, _: ChildStorageKey, _: ChildInfo, _: &[u8]) -> Option<Vec<u8>> {
-		panic!("original_child_storage_hash: unsupported feature for parachain validation")
-	}
-
 	fn child_storage(&self, _: ChildStorageKey, _: ChildInfo, _: &[u8]) -> Option<Vec<u8>> {
 		panic!("child_storage: unsupported feature for parachain validation")
 	}

runtime/test-runtime/client/src/lib.rs

@@ -319,5 +319,5 @@ pub fn new_light_fetcher() -> LightFetcher {
 
 /// Create a new native executor.
 pub fn new_native_executor() -> sc_executor::NativeExecutor<LocalExecutor> {
-	sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None)
+	sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None, 8)
 }

service/src/chain_spec.rs

@@ -19,7 +19,9 @@
 use sp_core::{Pair, Public, crypto::UncheckedInto, sr25519};
 use polkadot_primitives::{AccountId, AccountPublic, parachain::ValidatorId};
 use polkadot_runtime as polkadot;
-use polkadot_runtime::constants::currency::DOTS;
+use kusama_runtime as kusama;
+use polkadot::constants::currency::DOTS;
+use kusama::constants::currency::DOTS as KSM;
 use sc_chain_spec::ChainSpecExtension;
 use sp_runtime::{traits::IdentifyAccount, Perbill};
 use serde::{Serialize, Deserialize};
@@ -31,7 +33,8 @@ use im_online::sr25519::{AuthorityId as ImOnlineId};
 use authority_discovery_primitives::AuthorityId as AuthorityDiscoveryId;
 use pallet_staking::Forcing;
 
-const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
+const POLKADOT_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
+const KUSAMA_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
 const DEFAULT_PROTOCOL_ID: &str = "dot";
 
 /// Node `ChainSpec` extensions.
@@ -47,24 +50,27 @@ pub struct Extensions {
 	pub bad_blocks: sc_client::BadBlocks<polkadot_primitives::Block>,
 }
 
-/// The `ChainSpec`.
-///
-/// We use the same `ChainSpec` type for Polkadot and Kusama. As Kusama
-/// is only loaded from a file, the `GenesisConfig` type is not used.
-pub type ChainSpec = service::ChainSpec<
+/// The `ChainSpec parametrised for polkadot runtime`.
+pub type PolkadotChainSpec = service::GenericChainSpec<
 	polkadot::GenesisConfig,
 	Extensions,
 >;
 
-pub fn kusama_config() -> Result<ChainSpec, String> {
-	ChainSpec::from_json_bytes(&include_bytes!("../res/kusama.json")[..])
+/// The `ChainSpec parametrised for kusama runtime`.
+pub type KusamaChainSpec = service::GenericChainSpec<
+	kusama::GenesisConfig,
+	Extensions,
+>;
+
+pub fn kusama_config() -> Result<KusamaChainSpec, String> {
+	KusamaChainSpec::from_json_bytes(&include_bytes!("../res/kusama.json")[..])
 }
 
-pub fn westend_config() -> Result<ChainSpec, String> {
-	ChainSpec::from_json_bytes(&include_bytes!("../res/westend.json")[..])
+pub fn westend_config() -> Result<PolkadotChainSpec, String> {
+	PolkadotChainSpec::from_json_bytes(&include_bytes!("../res/westend.json")[..])
 }
 
-fn session_keys(
+fn polkadot_session_keys(
 	babe: BabeId,
 	grandpa: GrandpaId,
 	im_online: ImOnlineId,
@@ -74,7 +80,103 @@ fn session_keys(
 	polkadot::SessionKeys { babe, grandpa, im_online, parachain_validator, authority_discovery }
 }
 
-fn staging_testnet_config_genesis() -> polkadot::GenesisConfig {
+fn kusama_session_keys(
+	babe: BabeId,
+	grandpa: GrandpaId,
+	im_online: ImOnlineId,
+	parachain_validator: ValidatorId,
+	authority_discovery: AuthorityDiscoveryId
+) -> kusama::SessionKeys {
+	kusama::SessionKeys { babe, grandpa, im_online, parachain_validator, authority_discovery }
+}
+
+fn polkadot_staging_testnet_config_genesis() -> polkadot::GenesisConfig {
+	// subkey inspect "$SECRET"
+	let endowed_accounts = vec![];
+
+	let initial_authorities: Vec<(
+		AccountId,
+		AccountId,
+		BabeId,
+		GrandpaId,
+		ImOnlineId,
+		ValidatorId,
+		AuthorityDiscoveryId
+	)> = vec![];
+
+	const ENDOWMENT: u128 = 1_000_000 * DOTS;
+	const STASH: u128 = 100 * DOTS;
+
+	polkadot::GenesisConfig {
+		system: Some(polkadot::SystemConfig {
+			code: polkadot::WASM_BINARY.to_vec(),
+			changes_trie_config: Default::default(),
+		}),
+		balances: Some(polkadot::BalancesConfig {
+			balances: endowed_accounts.iter()
+				.map(|k: &AccountId| (k.clone(), ENDOWMENT))
+				.chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
+				.collect(),
+		}),
+		indices: Some(polkadot::IndicesConfig {
+			indices: vec![],
+		}),
+		session: Some(polkadot::SessionConfig {