Merge branch 'master' into td-mmr

.config/lingua.dic

@@ -85,4 +85,12 @@ WND/S
 XCM/S
 XCMP/M
 include/BG
-isolate/BG
\ No newline at end of file
+isolate/BG
+dispatchable/SM
+VMP/SM
+HMP/SM
+DMP/SM
+MQC/SM
+VRF/SM
+assignee/SM
+adversary/SM
\ No newline at end of file

.config/spellcheck.toml

@@ -7,6 +7,7 @@ extra_dictionaries = ["lingua.dic"]
 # `Type`'s
 # 5x
 # He tagged it as 'TheGreatestOfAllTimes'
-transform_regex = ["^'([^\\s])'$", "^[0-9]+(?:\\.[0-9]*)?x$", "^'s$", "^\\+$"]
+# Transforms'
+transform_regex = ["^'([^\\s])'$", "^[0-9]+(?:\\.[0-9]*)?x$", "^(.*)'$", "^\\+$"]
 allow_concatenation = true
 allow_dashes = true

Cargo.toml

@@ -6,7 +6,7 @@ path = "src/main.rs"
 name = "polkadot"
 description = "Implementation of a https://polkadot.network node in Rust based on the Substrate framework."
 license = "GPL-3.0-only"
-version = "0.8.27"
+version = "0.8.28"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 readme = "README.md"

README.md

@@ -155,7 +155,7 @@ Westend's WND tokens, see the faucet
 
 ## Hacking on Polkadot
 
-If you'd actually like hack on Polkadot, you can grab the source code and build it. Ensure you have
+If you'd actually like to hack on Polkadot, you can grab the source code and build it. Ensure you have
 Rust and the support software installed. This script will install or update Rust and install the
 required dependencies (this may take up to 30 minutes on Mac machines):
 

cli/Cargo.toml

 [package]
 name = "polkadot-cli"
-version = "0.8.27"
+version = "0.8.28"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot Relay-chain Client Node"
 edition = "2018"

cli/src/command.rs

@@ -249,7 +249,7 @@ pub fn run() -> Result<()> {
 			})?)
 		},
 		Some(Subcommand::ValidationWorker(cmd)) => {
-			let mut builder = sc_cli::GlobalLoggerBuilder::new("");
+			let mut builder = sc_cli::LoggerBuilder::new("");
 			builder.with_colors(false);
 			let _ = builder.init();
 

core-primitives/Cargo.toml

@@ -8,7 +8,8 @@ edition = "2018"
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-parity-scale-codec = { version = "1.3.6", default-features = false, features = [ "derive" ] }
+parity-scale-codec = { version = "2.0.0", default-features = false, features = [ "derive" ] }
+parity-util-mem = { version = "0.9.0", default-features = false, optional = true }
 
 [features]
 default = [ "std" ]
@@ -17,4 +18,5 @@ std = [
 	"sp-runtime/std",
 	"sp-std/std",
 	"parity-scale-codec/std",
+	"parity-util-mem",
 ]

core-primitives/src/lib.rs

@@ -20,8 +20,12 @@
 //!
 //! These core Polkadot types are used by the relay chain and the Parachains.
 
-use sp_runtime::{generic, MultiSignature, traits::{Verify, BlakeTwo256, IdentifyAccount}};
+use sp_runtime::{generic, MultiSignature, traits::{Verify, IdentifyAccount}};
 use parity_scale_codec::{Encode, Decode};
+#[cfg(feature = "std")]
+use parity_util_mem::MallocSizeOf;
+
+pub use sp_runtime::traits::{BlakeTwo256, Hash as HashT};
 
 /// The block number type used by Polkadot.
 /// 32-bits will allow for 136 years of blocks assuming 1 block per second.
@@ -57,6 +61,7 @@ pub type Hash = sp_core::H256;
 ///
 /// This type makes it easy to enforce that a hash is a candidate hash on the type level.
 #[derive(Clone, Copy, Encode, Decode, Hash, Eq, PartialEq, Debug, Default)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct CandidateHash(pub Hash);
 
 #[cfg(feature="std")]
@@ -103,6 +108,7 @@ pub type DownwardMessage = sp_std::vec::Vec<u8>;
 /// A wrapped version of `DownwardMessage`. The difference is that it has attached the block number when
 /// the message was sent.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct InboundDownwardMessage<BlockNumber = crate::BlockNumber> {
 	/// The block number at which this messages was put into the downward message queue.
 	pub sent_at: BlockNumber,
@@ -112,6 +118,7 @@ pub struct InboundDownwardMessage<BlockNumber = crate::BlockNumber> {
 
 /// An HRMP message seen from the perspective of a recipient.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct InboundHrmpMessage<BlockNumber = crate::BlockNumber> {
 	/// The block number at which this message was sent.
 	/// Specifically, it is the block number at which the candidate that sends this message was
@@ -123,6 +130,7 @@ pub struct InboundHrmpMessage<BlockNumber = crate::BlockNumber> {
 
 /// An HRMP message seen from the perspective of a sender.
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct OutboundHrmpMessage<Id> {
 	/// The para that will get this message in its downward message queue.
 	pub recipient: Id,

erasure-coding/Cargo.toml

 [package]
 name = "polkadot-erasure-coding"
-version = "0.8.27"
+version = "0.8.28"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
 [dependencies]
 primitives = { package = "polkadot-primitives", path = "../primitives" }
 reed_solomon = { package = "reed-solomon-erasure", version = "4.0.2" }
-parity-scale-codec = { version = "1.3.6", default-features = false, features = ["derive"] }
+parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 trie = { package = "sp-trie", git = "https://github.com/paritytech/substrate", branch = "master" }
 thiserror = "1.0.23"

node/core/approval-voting/Cargo.toml

@@ -6,13 +6,13 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
-parity-scale-codec = { version = "1.3.5", default-features = false, features = ["bit-vec", "derive"] }
+parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" }
 polkadot-overseer = { path = "../../overseer" }
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-primitives = { path = "../../primitives" }
-bitvec = "0.17.4"
+bitvec = "0.20.1"
 
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }

node/core/approval-voting/src/aux_schema/mod.rs

@@ -39,7 +39,7 @@ use polkadot_primitives::v1::{
 	ValidatorIndex, GroupIndex, CandidateReceipt, SessionIndex, CoreIndex,
 	BlockNumber, Hash, CandidateHash,
 };
-use sp_consensus_slots::SlotNumber;
+use sp_consensus_slots::Slot;
 use parity_scale_codec::{Encode, Decode};
 
 use std::collections::{BTreeMap, HashMap};
@@ -94,7 +94,7 @@ pub(crate) struct CandidateEntry {
 pub(crate) struct BlockEntry {
 	block_hash: Hash,
 	session: SessionIndex,
-	slot: SlotNumber,
+	slot: Slot,
 	relay_vrf_story: RelayVRF,
 	// The candidates included as-of this block and the index of the core they are
 	// leaving. Sorted ascending by core index.

node/core/approval-voting/src/aux_schema/tests.rs

@@ -89,7 +89,7 @@ fn make_block_entry(
 	BlockEntry {
 		block_hash,
 		session: 1,
-		slot: 1,
+		slot: 1.into(),
 		relay_vrf_story: RelayVRF([0u8; 32]),
 		approved_bitfield: make_bitvec(candidates.len()),
 		candidates,

node/core/av-store/Cargo.toml

@@ -7,14 +7,14 @@ edition = "2018"
 [dependencies]
 futures = "0.3.12"
 futures-timer = "3.0.2"
-kvdb = "0.8.0"
-kvdb-rocksdb = "0.10.0"
+kvdb = "0.9.0"
+kvdb-rocksdb = "0.11.0"
 thiserror = "1.0.23"
 tracing = "0.1.22"
 tracing-futures = "0.2.4"
-bitvec = "0.17.4"
+bitvec = "0.20.1"
 
-parity-scale-codec = { version = "1.3.6", features = ["derive"] }
+parity-scale-codec = { version = "2.0.0", features = ["derive"] }
 erasure = { package = "polkadot-erasure-coding", path = "../../../erasure-coding" }
 polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
@@ -27,7 +27,7 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master
 log = "0.4.13"
 env_logger = "0.8.2"
 assert_matches = "1.4.0"
-kvdb-memorydb = "0.8.0"
+kvdb-memorydb = "0.9.0"
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

node/core/av-store/src/lib.rs

@@ -133,16 +133,16 @@ impl Decode for BEBlockNumber {
 #[derive(Debug, Encode, Decode)]
 enum State {
 	/// Candidate data was first observed at the given time but is not available in any block.
-	#[codec(index = "0")]
+	#[codec(index = 0)]
 	Unavailable(BETimestamp),
 	/// The candidate was first observed at the given time and was included in the given list of unfinalized blocks, which may be
 	/// empty. The timestamp here is not used for pruning. Either one of these blocks will be finalized or the state will regress to
 	/// `State::Unavailable`, in which case the same timestamp will be reused. Blocks are sorted ascending first by block number and
 	/// then hash.
-	#[codec(index = "1")]
+	#[codec(index = 1)]
 	Unfinalized(BETimestamp, Vec<(BEBlockNumber, Hash)>),
 	/// Candidate data has appeared in a finalized block and did so at the given time.
-	#[codec(index = "2")]
+	#[codec(index = 2)]
 	Finalized(BETimestamp)
 }
 
@@ -967,7 +967,9 @@ fn process_message(
 		}
 		AvailabilityStoreMessage::QueryChunkAvailability(candidate, validator_index, tx) => {
 			let a = load_meta(&subsystem.db, &candidate)?
-				.map_or(false, |m| *m.chunks_stored.get(validator_index as usize).unwrap_or(&false));
+				.map_or(false, |m|
+					*m.chunks_stored.get(validator_index as usize).as_deref().unwrap_or(&false)
+				);
 			let _ = tx.send(a);
 		}
 		AvailabilityStoreMessage::StoreChunk {

node/core/av-store/src/tests.rs

@@ -103,11 +103,9 @@ impl Default for TestState {
 	fn default() -> Self {
 		let persisted_validation_data = PersistedValidationData {
 			parent_head: HeadData(vec![7, 8, 9]),
-			block_number: 5,
-			hrmp_mqc_heads: Vec::new(),
-			dmq_mqc_head: Default::default(),
+			relay_parent_number: 5,
 			max_pov_size: 1024,
-			relay_storage_root: Default::default(),
+			relay_parent_storage_root: Default::default(),
 		};
 
 		let pruning_config = PruningConfig {

node/core/backing/Cargo.toml

@@ -13,7 +13,7 @@ polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsys
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 erasure-coding = { package = "polkadot-erasure-coding", path = "../../../erasure-coding" }
 statement-table = { package = "polkadot-statement-table", path = "../../../statement-table" }
-bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] }
+bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] }
 tracing = "0.1.22"
 tracing-futures = "0.2.4"
 thiserror = "1.0.23"

node/core/backing/src/lib.rs

@@ -19,7 +19,6 @@
 #![deny(unused_crate_dependencies)]
 
 use std::collections::{HashMap, HashSet};
-use std::convert::TryFrom;
 use std::pin::Pin;
 use std::sync::Arc;
 
@@ -28,13 +27,12 @@ use futures::{channel::{mpsc, oneshot}, Future, FutureExt, SinkExt, StreamExt};
 
 use sp_keystore::SyncCryptoStorePtr;
 use polkadot_primitives::v1::{
-	CommittedCandidateReceipt, BackedCandidate, Id as ParaId, ValidatorId,
-	ValidatorIndex, SigningContext, PoV, CandidateHash,
-	CandidateDescriptor, AvailableData, ValidatorSignature, Hash, CandidateReceipt,
-	CoreState, CoreIndex, CollatorId, ValidityAttestation, CandidateCommitments,
+	AvailableData, BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash,
+	CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreIndex, CoreState, Hash, Id as ParaId,
+	PoV, SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation,
 };
 use polkadot_node_primitives::{
-	FromTableMisbehavior, Statement, SignedFullStatement, MisbehaviorReport, ValidationResult,
+	Statement, SignedFullStatement, ValidationResult,
 };
 use polkadot_subsystem::{
 	JaegerSpan, PerLeafSpan,
@@ -60,8 +58,9 @@ use statement_table::{
 	Context as TableContextTrait,
 	Table,
 	v1::{
+		SignedStatement as TableSignedStatement,
 		Statement as TableStatement,
-		SignedStatement as TableSignedStatement, Summary as TableSummary,
+		Summary as TableSummary,
 	},
 };
 use thiserror::Error;
@@ -145,8 +144,6 @@ struct CandidateBackingJob {
 	/// The candidates that are includable, by hash. Each entry here indicates
 	/// that we've sent the provisioner the backed candidate.
 	backed: HashSet<CandidateHash>,
-	/// We have already reported misbehaviors for these validators.
-	reported_misbehavior_for: HashSet<ValidatorIndex>,
 	keystore: SyncCryptoStorePtr,
 	table: Table<TableContext>,
 	table_context: TableContext,
@@ -644,36 +641,17 @@ impl CandidateBackingJob {
 	}
 
 	/// Check if there have happened any new misbehaviors and issue necessary messages.
-	///
-	/// TODO: Report multiple misbehaviors (https://github.com/paritytech/polkadot/issues/1387)
 	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn issue_new_misbehaviors(&mut self) -> Result<(), Error> {
-		let mut reports = Vec::new();
-
-		for (k, v) in self.table.get_misbehavior().iter() {
-			if !self.reported_misbehavior_for.contains(k) {
-				self.reported_misbehavior_for.insert(*k);
-
-				let f = FromTableMisbehavior {
-					id: *k,
-					report: v.clone(),
-					signing_context: self.table_context.signing_context.clone(),
-					key: self.table_context.validators[*k as usize].clone(),
-				};
-
-				if let Ok(report) = MisbehaviorReport::try_from(f) {
-					let message = ProvisionerMessage::ProvisionableData(
-						self.parent,
-						ProvisionableData::MisbehaviorReport(self.parent, report),
-					);
-
-					reports.push(message);
-				}
-			}
-		}
-
-		for report in reports.drain(..) {
-			self.send_to_provisioner(report).await?
+		// collect the misbehaviors to avoid double mutable self borrow issues
+		let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect();
+		for (validator_id, report) in misbehaviors {
+			self.send_to_provisioner(
+				ProvisionerMessage::ProvisionableData(
+					self.parent,
+					ProvisionableData::MisbehaviorReport(self.parent, validator_id, report)
+				)
+			).await?
 		}
 
 		Ok(())
@@ -1086,7 +1064,6 @@ impl util::JobTrait for CandidateBackingJob {
 				seconded: None,
 				unbacked_candidates: HashMap::new(),
 				backed: HashSet::new(),
-				reported_misbehavior_for: HashSet::new(),
 				keystore,
 				table: Table::default(),
 				table_context,
@@ -1199,9 +1176,7 @@ mod tests {
 	use super::*;
 	use assert_matches::assert_matches;
 	use futures::{future, Future};
-	use polkadot_primitives::v1::{
-		ScheduledCore, BlockData, PersistedValidationData, HeadData, GroupRotationInfo,
-	};
+	use polkadot_primitives::v1::{BlockData, GroupRotationInfo, HeadData, PersistedValidationData, ScheduledCore};
 	use polkadot_subsystem::{
 		messages::{RuntimeApiRequest, RuntimeApiMessage},
 		ActiveLeavesUpdate, FromOverseer, OverseerSignal,
@@ -1210,12 +1185,23 @@ mod tests {
 	use sp_keyring::Sr25519Keyring;
 	use sp_application_crypto::AppKey;
 	use sp_keystore::{CryptoStore, SyncCryptoStore};
+	use statement_table::v1::Misbehavior;
 	use std::collections::HashMap;
 
 	fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
 		val_ids.iter().map(|v| v.public().into()).collect()
 	}
 
+	fn table_statement_to_primitive(
+		statement: TableStatement,
+	) -> Statement {
+		match statement {
+			TableStatement::Candidate(committed_candidate_receipt) => Statement::Seconded(committed_candidate_receipt),
+			TableStatement::Valid(candidate_hash) => Statement::Valid(candidate_hash),
+			TableStatement::Invalid(candidate_hash) => Statement::Invalid(candidate_hash),
+		}
+	}
+
 	struct TestState {
 		chain_ids: Vec<ParaId>,
 		keystore: SyncCryptoStorePtr,
@@ -1288,11 +1274,9 @@ mod tests {
 
 			let validation_data = PersistedValidationData {
 				parent_head: HeadData(vec![7, 8, 9]),
-				block_number: Default::default(),
-				hrmp_mqc_heads: Vec::new(),
-				dmq_mqc_head: Default::default(),
+				relay_parent_number: Default::default(),
 				max_pov_size: 1024,
-				relay_storage_root: Default::default(),
+				relay_parent_storage_root: Default::default(),
 			};
 
 			Self {
@@ -1808,7 +1792,10 @@ mod tests {
 			assert!(candidates[0].validity_votes.contains(
 				&ValidityAttestation::Explicit(signed_c.signature().clone())
 			));
-			assert_eq!(candidates[0].validator_indices, bitvec::bitvec![Lsb0, u8; 1, 0, 1, 1]);
+			assert_eq!(
+				candidates[0].validator_indices,
+				bitvec::bitvec![bitvec::order::Lsb0, u8; 1, 0, 1, 1],
+			);
 
 			virtual_overseer.send(FromOverseer::Signal(
 				OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent)))
@@ -1950,19 +1937,30 @@ mod tests {
 						_,
 						ProvisionableData::MisbehaviorReport(
 							relay_parent,
-							MisbehaviorReport::SelfContradiction(_, s1, s2),
+							validator_index,
+							Misbehavior::ValidityDoubleVote(vdv),
 						)
 					)
 				) if relay_parent == test_state.relay_parent => {
-					s1.check_signature(
+					let ((t1, s1), (t2, s2)) = vdv.deconstruct::<TableContext>();
+					let t1 = table_statement_to_primitive(t1);
+					let t2 = table_statement_to_primitive(t2);
+
+					SignedFullStatement::new(
+						t1,
+						validator_index,
+						s1,
 						&test_state.signing_context,
-						&test_state.validator_public[s1.validator_index() as usize],
-					).unwrap();
+						&test_state.validator_public[validator_index as usize],
+					).expect("signature must be valid");
 
-					s2.check_signature(
+					SignedFullStatement::new(
+						t2,
+						validator_index,
+						s2,
 						&test_state.signing_context,
-						&test_state.validator_public[s2.validator_index() as usize],
-					).unwrap();
+						&test_state.validator_public[validator_index as usize],
+					).expect("signature must be valid");
 				}
 			);
 
@@ -1979,19 +1977,30 @@ mod tests {
 						_,
 						ProvisionableData::MisbehaviorReport(
 							relay_parent,
-							MisbehaviorReport::SelfContradiction(_, s1, s2),
+							validator_index,
+							Misbehavior::ValidityDoubleVote(vdv),
 						)
 					)
 				) if relay_parent == test_state.relay_parent => {
-					s1.check_signature(
+					let ((t1, s1), (t2, s2)) = vdv.deconstruct::<TableContext>();
+					let t1 = table_statement_to_primitive(t1);
+					let t2 = table_statement_to_primitive(t2);
+
+					SignedFullStatement::new(
+						t1,
+						validator_index,
+						s1,
 						&test_state.signing_context,
-						&test_state.validator_public[s1.validator_index() as usize],
-					).unwrap();
+						&test_state.validator_public[validator_index as usize],
+					).expect("signature must be valid");
 
-					s2.check_signature(
+					SignedFullStatement::new(
+						t2,
+						validator_index,
+						s2,
 						&test_state.signing_context,
-						&test_state.validator_public[s2.validator_index() as usize],
-					).unwrap();
+						&test_state.validator_public[validator_index as usize],
+					).expect("signature must be valid");
 				}
 			);
 		});
@@ -2464,6 +2473,7 @@ mod tests {
 	#[test]
 	fn candidate_backing_reorders_votes() {
 		use sp_core::Encode;
+		use std::convert::TryFrom;
 
 		let relay_parent = [1; 32].into();
 		let para_id = ParaId::from(10);

node/core/candidate-validation/Cargo.toml

@@ -10,7 +10,7 @@ tracing = "0.1.22"
 tracing-futures = "0.2.4"
 
 sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "master" }
-parity-scale-codec = { version = "1.3.6", default-features = false, features = ["bit-vec", "derive"] }
+parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-parachain = { path = "../../../parachain" }

node/core/candidate-validation/src/lib.rs

@@ -438,10 +438,8 @@ fn validate_candidate_exhaustive<B: ValidationBackend, S: SpawnNamed + 'static>(
 	let params = ValidationParams {
 		parent_head: persisted_validation_data.parent_head.clone(),
 		block_data: pov.block_data.clone(),
-		relay_chain_height: persisted_validation_data.block_number,
-		relay_storage_root: persisted_validation_data.relay_storage_root,
-		dmq_mqc_head: persisted_validation_data.dmq_mqc_head,
-		hrmp_mqc_heads: persisted_validation_data.hrmp_mqc_heads.clone(),
+		relay_parent_number: persisted_validation_data.relay_parent_number,
+		relay_parent_storage_root: persisted_validation_data.relay_parent_storage_root,
 	};
 
 	match B::validate(backend_arg, &validation_code, params, spawn) {