Update Substrate (#623)

* Update to latest Substrate master (#615)

* Update to latest Substrate master

* Remove unneeded patch + warning

* Update `Cargo.lock`

* Fix tests

* Update again

* Bump Substrate (#616)

* Update lock

* Fix

* Few fixes

* Bump to latest Substrate

* Fixes

* fix pre-tx-pool compilation

* more compilation fixes

* Updates for the injection period

- Liberal slash-refunding
- Instant unbonding

* *: Enable refactored authority discovery (#624)

* *: Enable authority discovery module

* *: List authority discovery id after parachain validator id

Make sure existing key types don't change their order by appending the
authority discovery id instead of injecting it between im online id and
parachain validator id.

* *: Gate authority discovery module behind feature flag

* cli/src/lib.rs: Fix warnings

* cli/src/lib.rs: Shorten line length

* Bump Substrate

* Bump Substrate

* Line widths

* Line widths again

* Revert bump.

Cargo.toml

@@ -4,7 +4,7 @@ path = "src/main.rs"
 
 [package]
 name = "polkadot"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"
 edition = "2018"
@@ -47,6 +47,3 @@ maintenance = { status = "actively-developed" }
 [profile.release]
 # Polkadot runtime requires unwinding.
 panic = "unwind"
-
-[patch.crates-io]
-zstd-sys = { git = "https://github.com/bkchr/zstd-rs.git", branch = "bkchr-export-include-paths2" }

availability-store/Cargo.toml

 [package]
 name = "polkadot-availability-store"
 description = "Persistent database for parachain data"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
@@ -11,6 +11,6 @@ parking_lot = "0.9.0"
 log = "0.4.8"
 codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
 substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
-kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }

availability-store/src/lib.rs

@@ -76,7 +76,15 @@ impl Store {
 	/// Create a new `Store` with given config on disk.
 	pub fn new(config: Config) -> io::Result<Self> {
 		let mut db_config = DatabaseConfig::with_columns(Some(columns::NUM_COLUMNS));
-		db_config.memory_budget = config.cache_size;
+
+		if let Some(cache_size) = config.cache_size {
+			let mut memory_budget = std::collections::HashMap::new();
+			for i in 0..columns::NUM_COLUMNS {
+				memory_budget.insert(Some(i), cache_size / columns::NUM_COLUMNS as usize);
+			}
+
+			db_config.memory_budget = memory_budget;
+		}
 
 		let path = config.path.to_str().ok_or_else(|| io::Error::new(
 			io::ErrorKind::Other,

cli/Cargo.toml

 [package]
 name = "polkadot-cli"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot node implementation in Rust."
 edition = "2018"

cli/src/lib.rs

@@ -87,20 +87,40 @@ struct ValidationWorkerCommand {
 	pub mem_id: String,
 }
 
+#[derive(Debug, StructOpt, Clone)]
+struct PolkadotSubParams {
+	#[structopt(long = "enable-authority-discovery")]
+	pub authority_discovery_enabled: bool,
+}
+
+cli::impl_augment_clap!(PolkadotSubParams);
+
 /// Parses polkadot specific CLI arguments and run the service.
 pub fn run<W>(worker: W, version: cli::VersionInfo) -> error::Result<()> where
 	W: Worker,
 {
-	match cli::parse_and_prepare::<PolkadotSubCommands, NoCustom, _>(&version, "parity-polkadot", std::env::args()) {
+	match cli::parse_and_prepare::<PolkadotSubCommands, PolkadotSubParams, _>(
+		&version,
+		"parity-polkadot",
+		std::env::args(),
+	) {
 		cli::ParseAndPrepare::Run(cmd) => cmd.run(load_spec, worker,
-		|worker, _cli_args, _custom_args, mut config| {
+		|worker, _cli_args, custom_args, mut config| {
 			info!("{}", version.name);
 			info!("  version {}", config.full_version());
 			info!("  by {}, 2017-2019", version.author);
 			info!("Chain specification: {}", config.chain_spec.name());
+			if config.chain_spec.name().starts_with("Kusama") {
+				info!("----------------------------");
+				info!("This chain is not in any way");
+				info!("      endorsed by the       ");
+				info!("     KUSAMA FOUNDATION      ");
+				info!("----------------------------");
+			}
 			info!("Node name: {}", config.name);
 			info!("Roles: {}", display_role(&config));
 			config.custom = worker.configuration();
+			config.custom.authority_discovery_enabled = custom_args.authority_discovery_enabled;
 			let runtime = Runtime::new().map_err(|e| format!("{:?}", e))?;
 			match config.roles {
 				service::Roles::LIGHT =>

collator/Cargo.toml

 [package]
 name = "polkadot-collator"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Collator node implementation"
 edition = "2018"

erasure-coding/Cargo.toml

 [package]
 name = "polkadot-erasure-coding"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 

executor/Cargo.toml

 [package]
 name = "polkadot-executor"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot node implementation in Rust."
 edition = "2018"

network/Cargo.toml

 [package]
 name = "polkadot-network"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot-specific networking protocol"
 edition = "2018"
@@ -20,6 +20,7 @@ futures03 = { package = "futures", version = "0.3.1", features = ["compat"] }
 log = "0.4.8"
 exit-future = "0.1.4"
 substrate-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 
 [dev-dependencies]
 substrate-keyring = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

network/src/gossip.rs

@@ -50,7 +50,7 @@
 //! will be noted as non-beneficial to Substrate's peer-set management utility.
 
 use sr_primitives::{generic::BlockId, traits::ProvideRuntimeApi};
-use substrate_client::error::Error as ClientError;
+use sp_blockchain::Error as ClientError;
 use substrate_network::{config::Roles, PeerId};
 use substrate_network::consensus_gossip::{
 	self as network_gossip, ValidationResult as GossipValidationResult,
@@ -792,9 +792,10 @@ mod tests {
 
 		{
 			let mut message_allowed = validator.message_allowed();
-			assert!(message_allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &topic_a, &encoded));
-			assert!(!message_allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &topic_b, &encoded));
-			assert!(!message_allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &topic_c, &encoded));
+			let intent = MessageIntent::Broadcast;
+			assert!(message_allowed(&peer_a, intent, &topic_a, &encoded));
+			assert!(!message_allowed(&peer_a, intent, &topic_b, &encoded));
+			assert!(!message_allowed(&peer_a, intent, &topic_c, &encoded));
 		}
 	}
 
@@ -900,7 +901,7 @@ mod tests {
 
 		{
 			let mut message_allowed = validator.message_allowed();
-			assert!(!message_allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &topic_a, &encoded[..]));
+			assert!(!message_allowed(&peer_a, MessageIntent::Broadcast, &topic_a, &encoded[..]));
 		}
 
 		validator
@@ -913,7 +914,7 @@ mod tests {
 			.note_aware_under_leaf(&hash_a, c_hash);
 		{
 			let mut message_allowed = validator.message_allowed();
-			assert!(message_allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &topic_a, &encoded[..]));
+			assert!(message_allowed(&peer_a, MessageIntent::Broadcast, &topic_a, &encoded[..]));
 		}
 	}
 
@@ -1006,8 +1007,9 @@ mod tests {
 			}).encode();
 
 			let mut allowed = validator.inner.message_allowed();
-			assert!(allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
-			assert!(!allowed(&peer_b, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
+			let intent = MessageIntent::Broadcast;
+			assert!(allowed(&peer_a, intent, &root_a_topic, &message[..]));
+			assert!(!allowed(&peer_b, intent, &root_a_topic, &message[..]));
 		}
 	}
 
@@ -1077,8 +1079,9 @@ mod tests {
 			}).encode();
 
 			let mut allowed = validator.inner.message_allowed();
-			assert!(!allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
-			assert!(!allowed(&peer_b, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
+			let intent = MessageIntent::Broadcast;
+			assert!(!allowed(&peer_a, intent, &root_a_topic, &message[..]));
+			assert!(!allowed(&peer_b, intent, &root_a_topic, &message[..]));
 		}
 
 		// peer A gets updated to the chain head. now we'll attempt to broadcast
@@ -1115,8 +1118,9 @@ mod tests {
 			}).encode();
 
 			let mut allowed = validator.inner.message_allowed();
-			assert!(allowed(&peer_a, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
-			assert!(!allowed(&peer_b, MessageIntent::Broadcast { previous_attempts: 0 }, &root_a_topic, &message[..]));
+			let intent = MessageIntent::Broadcast;
+			assert!(allowed(&peer_a, intent, &root_a_topic, &message[..]));
+			assert!(!allowed(&peer_b, intent, &root_a_topic, &message[..]));
 		}
 	}
 

network/src/gossip/message_routing.rs

@@ -63,7 +63,7 @@
 use sr_primitives::traits::{BlakeTwo256, Hash as HashT};
 use polkadot_primitives::Hash;
 use std::collections::{HashMap, HashSet};
-use substrate_client::error::Error as ClientError;
+use sp_blockchain::Error as ClientError;
 use super::{MAX_CHAIN_HEADS, GossipValidationResult, LeavesVec, ChainContext};
 
 /// Construct a topic for a message queue root deterministically.

network/src/router.rs

@@ -126,7 +126,7 @@ impl<P, E: Clone, N: NetworkService, T: Clone> Clone for Router<P, E, N, T> {
 }
 
 impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> where
-	P::Api: ParachainHost<Block, Error = substrate_client::error::Error>,
+	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 	N: NetworkService,
 	T: Clone + Executor + Send + 'static,
 	E: Future<Item=(),Error=()> + Clone + Send + 'static,

network/src/tests/mod.rs

@@ -89,7 +89,7 @@ impl crate::gossip::ChainContext for TestChainContext {
 	}
 
 	fn leaf_unrouted_roots(&self, leaf: &Hash, with_queue_root: &mut dyn FnMut(&Hash))
-		-> Result<(), substrate_client::error::Error>
+		-> Result<(), sp_blockchain::Error>
 	{
 		for root in self.ingress_roots.get(leaf).into_iter().flat_map(|roots| roots) {
 			with_queue_root(root)

network/src/tests/validation.rs

@@ -33,7 +33,7 @@ use polkadot_primitives::parachain::{
 	FeeSchedule, HeadData, Retriable, CollatorId
 };
 use parking_lot::Mutex;
-use substrate_client::error::Result as ClientResult;
+use sp_blockchain::Result as ClientResult;
 use sr_api::{Core, RuntimeVersion, StorageProof, ApiExt};
 use sr_primitives::traits::{ApiRef, ProvideRuntimeApi};
 
@@ -231,7 +231,7 @@ impl Core<Block> for RuntimeApi {
 }
 
 impl ApiExt<Block> for RuntimeApi {
-	type Error = substrate_client::error::Error;
+	type Error = sp_blockchain::Error;
 
 	fn map_api_result<F: FnOnce(&Self) -> Result<R, E>, R, E>(
 		&self,

network/src/validation.rs

@@ -205,7 +205,7 @@ impl<P, E, N, T> ValidationNetwork<P, E, N, T> where N: NetworkService {
 /// A long-lived network which can create parachain statement  routing processes on demand.
 impl<P, E, N, T> ParachainNetwork for ValidationNetwork<P, E, N, T> where
 	P: ProvideRuntimeApi + Send + Sync + 'static,
-	P::Api: ParachainHost<Block, Error = substrate_client::error::Error>,
+	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 	E: Clone + Future<Item=(),Error=()> + Send + Sync + 'static,
 	N: NetworkService,
 	T: Clone + Executor + Send + Sync + 'static,

parachain/Cargo.toml

 [package]
 name = "polkadot-parachain"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Types and utilities for creating and working with parachains"
 edition = "2018"

primitives/Cargo.toml

 [package]
 name = "polkadot-primitives"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 

rpc/Cargo.toml

 [package]
 name = "polkadot-rpc"
-version = "0.6.17"
+version = "0.7.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
@@ -10,8 +10,8 @@ jsonrpc-core = "14.0.3"
 polkadot-primitives = { path = "../primitives" }
 sr-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master"  }
 substrate-rpc = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
-transaction_pool = { package = "substrate-transaction-pool", git = "https://github.com/paritytech/substrate", branch = "polkadot-master"  }
-frame-system-rpc = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master"  }
+txpool-api = { package = "sp-transaction-pool-api", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+frame-rpc-system = { package = "substrate-frame-rpc-system", git = "https://github.com/paritytech/substrate", branch = "polkadot-master"  }
 pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 polkadot-runtime = { path = "../runtime" }
 

rpc/src/lib.rs

@@ -22,30 +22,55 @@ use std::sync::Arc;
 
 use polkadot_primitives::{Block, AccountId, Nonce, Balance};
 use sr_primitives::traits::ProvideRuntimeApi;
-use transaction_pool::txpool::{ChainApi, Pool};
+use txpool_api::TransactionPool;
 use polkadot_runtime::UncheckedExtrinsic;
 
 /// A type representing all RPC extensions.
 pub type RpcExtension = jsonrpc_core::IoHandler<substrate_rpc::Metadata>;
 
 /// Instantiate all RPC extensions.
-pub fn create<C, P>(client: Arc<C>, pool: Arc<Pool<P>>) -> RpcExtension where
+pub fn create_full<C, P>(client: Arc<C>, pool: Arc<P>) -> RpcExtension where
 	C: ProvideRuntimeApi,
 	C: client::blockchain::HeaderBackend<Block>,
 	C: Send + Sync + 'static,
-	C::Api: frame_system_rpc::AccountNonceApi<Block, AccountId, Nonce>,
+	C::Api: frame_rpc_system::AccountNonceApi<Block, AccountId, Nonce>,
 	C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance, UncheckedExtrinsic>,
-	P: ChainApi + Sync + Send + 'static,
+	P: TransactionPool + Sync + Send + 'static,
 {
-	use frame_system_rpc::{System, SystemApi};
+	use frame_rpc_system::{FullSystem, SystemApi};
 	use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
 
 	let mut io = jsonrpc_core::IoHandler::default();
 	io.extend_with(
-		SystemApi::to_delegate(System::new(client.clone(), pool))
+		SystemApi::to_delegate(FullSystem::new(client.clone(), pool))
 	);
 	io.extend_with(
 		TransactionPaymentApi::to_delegate(TransactionPayment::new(client))
 	);
 	io
 }
+
+/// Instantiate all RPC extensions for light node.
+pub fn create_light<C, P, F>(
+	client: Arc<C>,
+	remote_blockchain: Arc<dyn client::light::blockchain::RemoteBlockchain<Block>>,
+	fetcher: Arc<F>,
+	pool: Arc<P>,
+) -> RpcExtension
+	where
+		C: ProvideRuntimeApi,
+		C: client::blockchain::HeaderBackend<Block>,
+		C: Send + Sync + 'static,
+		C::Api: frame_rpc_system::AccountNonceApi<Block, AccountId, Nonce>,
+		C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance, UncheckedExtrinsic>,
+		P: TransactionPool + Sync + Send + 'static,
+		F: client::light::fetcher::Fetcher<Block> + 'static,
+{
+	use frame_rpc_system::{LightSystem, SystemApi};
+
+	let mut io = jsonrpc_core::IoHandler::default();
+	io.extend_with(
+		SystemApi::<AccountId, Nonce>::to_delegate(LightSystem::new(client, remote_blockchain, fetcher, pool))
+	);
+	io
+}