Unverified Commit 96af6ead authored by asynchronous rob's avatar asynchronous rob Committed by GitHub
Browse files

Refactor primitives (#1383)

* create a v1 primitives module

* Improve guide on availability types

* punctuate

* new parachains runtime uses new primitives

* tests of new runtime now use new primitives

* add ErasureChunk to guide

* export erasure chunk from v1 primitives

* subsystem crate uses v1 primitives

* node-primitives uses new v1 primitives

* port overseer to new primitives

* new-proposer uses v1 primitives (no ParachainHost anymore)

* fix no-std compilation for primitives

* service-new uses v1 primitives

* network-bridge uses new primitives

* statement distribution uses v1 primitives

* PoV distribution uses v1 primitives; add PoV::hash fn

* move parachain to v0

* remove inclusion_inherent module and place into v1

* remove everything from primitives crate root

* remove some unused old types from v0 primitives

* point everything else at primitives::v0

* squanch some warns up

* add RuntimeDebug import to no-std as well

* port over statement-table and validation

* fix final errors in validation and node-primitives

* add dummy Ord impl to committed candidate receipt

* guide: update CandidateValidationMessage

* add primitive for validationoutputs

* expand CandidateValidationMessage further

* bikeshed

* add some impls to omitted-validation-data and available-data

* expand CandidateValidationMessage

* make erasure-coding generic over v1/v0

* update usages of erasure-coding

* implement commitments.hash()

* use Arc<Pov> for CandidateValidation

* improve new erasure-coding method names

* fix up candidate backing

* update docs a bit

* fix most tests and add short-circuiting to make_pov_available

* fix remainder of candidate backing tests

* squanching warns

* squanch it up

* some fallout

* overseer fallout

* free from polkadot-test-service hell
parent 8845df22
Pipeline #100069 passed with stages
in 25 minutes and 41 seconds
......@@ -25,12 +25,10 @@
use futures::prelude::*;
use futures::channel::{mpsc, oneshot};
use keystore::KeyStorePtr;
use polkadot_primitives::{
use polkadot_primitives::v0::{
Hash, Block,
parachain::{
PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
ParachainHost, AvailableData, OmittedValidationData,
},
PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
ParachainHost, AvailableData, OmittedValidationData,
};
use sp_runtime::traits::HashFor;
use sp_blockchain::Result as ClientResult;
......
......@@ -19,11 +19,8 @@ use kvdb_rocksdb::{Database, DatabaseConfig};
use kvdb::{KeyValueDB, DBTransaction};
use codec::{Encode, Decode};
use polkadot_erasure_coding as erasure;
use polkadot_primitives::{
Hash,
parachain::{
ErasureChunk, AvailableData, AbridgedCandidateReceipt,
},
use polkadot_primitives::v0::{
Hash, ErasureChunk, AvailableData, AbridgedCandidateReceipt,
};
use parking_lot::Mutex;
......@@ -273,7 +270,7 @@ impl Store {
// If there are no block data in the store at this point,
// check that they can be reconstructed now and add them to store if they can.
if self.execution_data(&candidate_hash).is_none() {
if let Ok(available_data) = erasure::reconstruct(
if let Ok(available_data) = erasure::reconstruct_v0(
n_validators as usize,
v.iter().map(|chunk| (chunk.chunk.as_ref(), chunk.index as usize)),
)
......@@ -390,7 +387,7 @@ impl Store {
mod tests {
use super::*;
use polkadot_erasure_coding::{self as erasure};
use polkadot_primitives::parachain::{
use polkadot_primitives::v0::{
Id as ParaId, BlockData, AvailableData, PoVBlock, OmittedValidationData,
};
......@@ -489,7 +486,7 @@ mod tests {
let available_data = available_data(&[42; 8]);
let n_validators = 5;
let erasure_chunks = erasure::obtain_chunks(
let erasure_chunks = erasure::obtain_chunks_v0(
n_validators,
&available_data,
).unwrap();
......
......@@ -33,8 +33,8 @@ use consensus_common::{
import_queue::CacheKeyId,
};
use sp_core::traits::SpawnNamed;
use polkadot_primitives::{Block, BlockId, Hash};
use polkadot_primitives::parachain::{
use polkadot_primitives::v0::{
Block, BlockId, Hash,
ParachainHost, ValidatorId, AbridgedCandidateReceipt, AvailableData,
ValidatorPair, ErasureChunk,
};
......
......@@ -55,12 +55,11 @@ use log::warn;
use sc_client_api::{StateBackend, BlockchainEvents};
use sp_blockchain::HeaderBackend;
use sp_core::Pair;
use polkadot_primitives::{
use polkadot_primitives::v0::{
BlockId, Hash, Block, DownwardMessage,
parachain::{
self, BlockData, DutyRoster, HeadData, Id as ParaId,
PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
}
BlockData, DutyRoster, HeadData, Id as ParaId,
PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
Collation, CollationInfo, collator_signature_payload,
};
use polkadot_cli::{
ProvideRuntimeApi, ParachainHost, IdentifyVariant,
......@@ -69,7 +68,7 @@ use polkadot_cli::{
pub use polkadot_cli::service::Configuration;
pub use polkadot_cli::Cli;
pub use polkadot_validation::SignedStatement;
pub use polkadot_primitives::parachain::CollatorId;
pub use polkadot_primitives::v0::CollatorId;
pub use sc_network::PeerId;
pub use service::RuntimeApiCollection;
pub use sc_cli::SubstrateCli;
......@@ -164,7 +163,7 @@ pub async fn collate<P>(
downward_messages: Vec<DownwardMessage>,
mut para_context: P,
key: Arc<CollatorPair>,
) -> Option<parachain::Collation>
) -> Option<Collation>
where
P: ParachainContext,
P::ProduceCandidate: Send,
......@@ -181,13 +180,13 @@ pub async fn collate<P>(
};
let pov_block_hash = pov_block.hash();
let signature = key.sign(&parachain::collator_signature_payload(
let signature = key.sign(&collator_signature_payload(
&relay_parent,
&local_id,
&pov_block_hash,
));
let info = parachain::CollationInfo {
let info = CollationInfo {
parachain_index: local_id,
relay_parent,
collator: key.public(),
......@@ -196,7 +195,7 @@ pub async fn collate<P>(
pov_block_hash,
};
let collation = parachain::Collation {
let collation = Collation {
info,
pov: pov_block,
};
......@@ -456,7 +455,7 @@ where
#[cfg(not(feature = "service-rewr"))]
fn compute_targets(para_id: ParaId, session_keys: &[ValidatorId], roster: DutyRoster) -> HashSet<ValidatorId> {
use polkadot_primitives::parachain::Chain;
use polkadot_primitives::v0::Chain;
roster.validator_duty.iter().enumerate()
.filter(|&(_, c)| c == &Chain::Parachain(para_id))
......
......@@ -94,3 +94,8 @@ pub enum DownwardMessage<AccountId = crate::AccountId> {
/// XCMP message for the Parachain.
XCMPMessage(sp_std::vec::Vec<u8>),
}
/// V1 primitives.
pub mod v1 {
pub use super::*;
}
......@@ -26,8 +26,8 @@
use codec::{Encode, Decode};
use reed_solomon::galois_16::{self, ReedSolomon};
use primitives::{Hash as H256, BlakeTwo256, HashT};
use primitives::parachain::AvailableData;
use primitives::v0::{self, Hash as H256, BlakeTwo256, HashT};
use primitives::v1;
use sp_core::Blake2Hasher;
use trie::{EMPTY_PREFIX, MemoryDB, Trie, TrieMut, trie_types::{TrieDBMut, TrieDB}};
......@@ -124,14 +124,32 @@ fn code_params(n_validators: usize) -> Result<CodeParams, Error> {
})
}
/// Obtain erasure-coded chunks for v0 `AvailableData`, one for each validator.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn obtain_chunks_v0(n_validators: usize, data: &v0::AvailableData)
-> Result<Vec<Vec<u8>>, Error>
{
obtain_chunks(n_validators, data)
}
/// Obtain erasure-coded chunks for v1 `AvailableData`, one for each validator.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn obtain_chunks_v1(n_validators: usize, data: &v1::AvailableData)
-> Result<Vec<Vec<u8>>, Error>
{
obtain_chunks(n_validators, data)
}
/// Obtain erasure-coded chunks, one for each validator.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData)
fn obtain_chunks<T: Encode>(n_validators: usize, data: &T)
-> Result<Vec<Vec<u8>>, Error>
{
let params = code_params(n_validators)?;
let encoded = available_data.encode();
let encoded = data.encode();
if encoded.is_empty() {
return Err(Error::BadPayload);
......@@ -145,15 +163,42 @@ pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData)
Ok(shards.into_iter().map(|w| w.into_inner()).collect())
}
/// Reconstruct the block data from a set of chunks.
/// Reconstruct the v0 available data from a set of chunks.
///
/// Provide an iterator containing chunk data and the corresponding index.
/// The indices of the present chunks must be indicated. If too few chunks
/// are provided, recovery is not possible.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn reconstruct_v0<'a, I: 'a>(n_validators: usize, chunks: I)
-> Result<v0::AvailableData, Error>
where I: IntoIterator<Item=(&'a [u8], usize)>
{
reconstruct(n_validators, chunks)
}
/// Reconstruct the v1 available data from a set of chunks.
///
/// Provide an iterator containing chunk data and the corresponding index.
/// The indices of the present chunks must be indicated. If too few chunks
/// are provided, recovery is not possible.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn reconstruct_v1<'a, I: 'a>(n_validators: usize, chunks: I)
-> Result<v1::AvailableData, Error>
where I: IntoIterator<Item=(&'a [u8], usize)>
{
reconstruct(n_validators, chunks)
}
/// Reconstruct decodable data from a set of chunks.
///
/// Provide an iterator containing chunk data and the corresponding index.
/// The indices of the present chunks must be indicated. If too few chunks
/// are provided, recovery is not possible.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn reconstruct<'a, I: 'a>(n_validators: usize, chunks: I)
-> Result<AvailableData, Error>
fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
where I: IntoIterator<Item=(&'a [u8], usize)>
{
let params = code_params(n_validators)?;
......@@ -343,7 +388,7 @@ impl<'a, I: Iterator<Item=&'a [u8]>> codec::Input for ShardInput<'a, I> {
#[cfg(test)]
mod tests {
use super::*;
use primitives::parachain::{BlockData, PoVBlock};
use primitives::v0::{AvailableData, BlockData, PoVBlock};
#[test]
fn field_order_is_right_size() {
......@@ -420,7 +465,7 @@ mod tests {
assert_eq!(chunks.len(), 10);
// any 4 chunks should work.
let reconstructed = reconstruct(
let reconstructed: AvailableData = reconstruct(
10,
[
(&*chunks[1], 1),
......
......@@ -17,8 +17,7 @@
//! Bridge between the network and consensus service for getting collations to it.
use codec::{Encode, Decode};
use polkadot_primitives::Hash;
use polkadot_primitives::parachain::{CollatorId, Id as ParaId, Collation};
use polkadot_primitives::v0::{Hash, CollatorId, Id as ParaId, Collation};
use sc_network::PeerId;
use futures::channel::oneshot;
......@@ -236,7 +235,7 @@ impl CollatorPool {
mod tests {
use super::*;
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::parachain::{CollationInfo, BlockData, PoVBlock};
use polkadot_primitives::v0::{CollationInfo, BlockData, PoVBlock};
use futures::executor::block_on;
fn make_pov(block_data: Vec<u8>) -> PoVBlock {
......
......@@ -33,7 +33,7 @@
use sc_network_gossip::{ValidationResult as GossipValidationResult};
use sc_network::ReputationChange;
use polkadot_validation::GenericStatement;
use polkadot_primitives::Hash;
use polkadot_primitives::v0::Hash;
use std::collections::HashMap;
......
......@@ -58,8 +58,8 @@ use sc_network_gossip::{
ValidatorContext, MessageIntent,
};
use polkadot_validation::{SignedStatement};
use polkadot_primitives::{Block, Hash};
use polkadot_primitives::parachain::{
use polkadot_primitives::v0::{
Block, Hash,
ParachainHost, ValidatorId, ErasureChunk as PrimitiveChunk, SigningContext, PoVBlock,
};
use polkadot_erasure_coding::{self as erasure};
......@@ -755,7 +755,7 @@ mod tests {
use sc_network_gossip::Validator as ValidatorT;
use std::sync::mpsc;
use parking_lot::Mutex;
use polkadot_primitives::parachain::{AbridgedCandidateReceipt, BlockData};
use polkadot_primitives::v0::{AbridgedCandidateReceipt, BlockData};
use sp_core::sr25519::Signature as Sr25519Signature;
use polkadot_validation::GenericStatement;
......
......@@ -19,7 +19,7 @@
//! Collations are attempted to be repropagated when a new validator connects,
//! a validator changes his session key, or when they are generated.
use polkadot_primitives::{Hash, parachain::{ValidatorId}};
use polkadot_primitives::v0::{Hash, ValidatorId};
use crate::legacy::collator_pool::Role;
use std::collections::{HashMap, HashSet};
use std::time::Duration;
......@@ -144,7 +144,7 @@ impl<C: Clone> LocalCollations<C> {
mod tests {
use super::*;
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::parachain::ValidatorId;
use polkadot_primitives::v0::ValidatorId;
#[test]
fn add_validator_with_ready_collation() {
......
......@@ -25,7 +25,7 @@ pub mod gossip;
use codec::Decode;
use futures::prelude::*;
use polkadot_primitives::Hash;
use polkadot_primitives::v0::Hash;
use sc_network::PeerId;
use sc_network_gossip::TopicNotification;
use log::debug;
......
......@@ -21,7 +21,7 @@
#![recursion_limit="256"]
use polkadot_primitives::{Block, Hash, BlakeTwo256, HashT};
use polkadot_primitives::v0::{Block, Hash, BlakeTwo256, HashT};
pub mod legacy;
pub mod protocol;
......
......@@ -30,12 +30,10 @@ use futures::task::{Context, Poll};
use futures::stream::{FuturesUnordered, StreamFuture};
use log::{debug, trace};
use polkadot_primitives::{
use polkadot_primitives::v0::{
Hash, Block,
parachain::{
PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
},
PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
};
use polkadot_validation::{
SharedTable, TableRouter, Network as ParachainNetwork, Validated, GenericStatement, Collators,
......
......@@ -17,8 +17,8 @@ use super::*;
use crate::legacy::gossip::GossipPoVBlock;
use parking_lot::Mutex;
use polkadot_primitives::Block;
use polkadot_primitives::parachain::{
use polkadot_primitives::v0::{
Block,
Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId,
Retriable, CollatorId, AbridgedCandidateReceipt,
GlobalValidationSchedule, LocalValidationData, ErasureChunk, SigningContext,
......@@ -198,7 +198,7 @@ sp_api::mock_impl_runtime_apis! {
parent_hash: Default::default(),
}
}
fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::DownwardMessage> {
fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::v0::DownwardMessage> {
Vec::new()
}
}
......
This diff is collapsed.
......@@ -2,9 +2,7 @@ use futures::prelude::*;
use futures::select;
use polkadot_node_subsystem::{messages::{AllMessages, ProvisionerInherentData, ProvisionerMessage}, SubsystemError};
use polkadot_overseer::OverseerHandler;
use polkadot_primitives::{
inclusion_inherent,
parachain::ParachainHost,
use polkadot_primitives::v1::{
Block, Hash, Header,
};
use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
......@@ -53,7 +51,7 @@ where
+ Send
+ Sync,
Client::Api:
ParachainHost<Block> + BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
Backend:
'static + sc_client_api::Backend<Block, State = sp_api::StateBackendFor<Client, Block>>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
......@@ -104,7 +102,7 @@ where
+ Send
+ Sync,
Client::Api:
ParachainHost<Block> + BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
Backend:
'static + sc_client_api::Backend<Block, State = sp_api::StateBackendFor<Client, Block>>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
......@@ -155,7 +153,7 @@ where
+ Send
+ Sync,
Client::Api:
ParachainHost<Block> + BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
Backend:
'static + sc_client_api::Backend<Block, State = sp_api::StateBackendFor<Client, Block>>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
......@@ -186,7 +184,7 @@ where
};
inherent_data.put_data(
inclusion_inherent::INHERENT_IDENTIFIER,
polkadot_primitives::v1::INCLUSION_INHERENT_IDENTIFIER,
&provisioner_data,
)?;
......
......@@ -33,7 +33,7 @@ use polkadot_subsystem::{
};
use polkadot_subsystem::messages::{NetworkBridgeEvent, NetworkBridgeMessage, AllMessages};
use node_primitives::{ProtocolId, View};
use polkadot_primitives::{Block, Hash};
use polkadot_primitives::v1::{Block, Hash};
use std::collections::btree_map::{BTreeMap, Entry as BEntry};
use std::collections::hash_map::{HashMap, Entry as HEntry};
......
......@@ -19,8 +19,7 @@
//! This is a gossip implementation of code that is responsible for distributing PoVs
//! among validators.
use polkadot_primitives::Hash;
use polkadot_primitives::parachain::{PoVBlock as PoV, CandidateDescriptor};
use polkadot_primitives::v1::{Hash, PoV, CandidateDescriptor};
use polkadot_subsystem::{
OverseerSignal, SubsystemContext, Subsystem, SubsystemResult, FromOverseer, SpawnedSubsystem,
};
......@@ -550,7 +549,7 @@ async fn run(
mod tests {
use super::*;
use futures::executor::{self, ThreadPool};
use polkadot_primitives::parachain::BlockData;
use polkadot_primitives::v1::BlockData;
use assert_matches::assert_matches;
fn make_pov(data: Vec<u8>) -> PoV {
......
......@@ -29,9 +29,8 @@ use polkadot_subsystem::messages::{
RuntimeApiRequest,
};
use node_primitives::{ProtocolId, View, SignedFullStatement};
use polkadot_primitives::Hash;
use polkadot_primitives::parachain::{
CompactStatement, ValidatorIndex, ValidatorId, SigningContext, ValidatorSignature,
use polkadot_primitives::v1::{
Hash, CompactStatement, ValidatorIndex, ValidatorId, SigningContext, ValidatorSignature,
};
use parity_scale_codec::{Encode, Decode};
......@@ -891,7 +890,7 @@ mod tests {
use super::*;
use sp_keyring::Sr25519Keyring;
use node_primitives::Statement;
use polkadot_primitives::parachain::{AbridgedCandidateReceipt};
use polkadot_primitives::v1::CommittedCandidateReceipt;
use assert_matches::assert_matches;
use futures::executor::{self, ThreadPool};
......@@ -911,23 +910,23 @@ mod tests {
};
let candidate_a = {
let mut c = AbridgedCandidateReceipt::default();
c.relay_parent = parent_hash;
c.parachain_index = 1.into();
let mut c = CommittedCandidateReceipt::default();
c.descriptor.relay_parent = parent_hash;
c.descriptor.para_id = 1.into();
c
};
let candidate_b = {
let mut c = AbridgedCandidateReceipt::default();
c.relay_parent = parent_hash;
c.parachain_index = 2.into();
let mut c = CommittedCandidateReceipt::default();
c.descriptor.relay_parent = parent_hash;
c.descriptor.para_id = 2.into();
c
};
let candidate_c = {
let mut c = AbridgedCandidateReceipt::default();
c.relay_parent = parent_hash;
c.parachain_index = 3.into();
let mut c = CommittedCandidateReceipt::default();
c.descriptor.relay_parent = parent_hash;
c.descriptor.para_id = 3.into();
c
};
......@@ -1140,9 +1139,9 @@ mod tests {
let hash_c = [3; 32].into();
let candidate = {
let mut c = AbridgedCandidateReceipt::default();
c.relay_parent = hash_c;
c.parachain_index = 1.into();
let mut c = CommittedCandidateReceipt::default();
c.descriptor.relay_parent = hash_c;
c.descriptor.para_id = 1.into();
c
};
let candidate_hash = candidate.hash();
......@@ -1275,9 +1274,9 @@ mod tests {
let hash_c = [3; 32].into();
let candidate = {
let mut c = AbridgedCandidateReceipt::default();
c.relay_parent = hash_b;
c.parachain_index = 1.into();
let mut c = CommittedCandidateReceipt::default();
c.descriptor.relay_parent = hash_b;
c.descriptor.para_id = 1.into();
c
};
......
......@@ -27,11 +27,11 @@ use futures::{
use futures_timer::Delay;
use kv_log_macro as log;
use polkadot_primitives::parachain::{BlockData, PoVBlock};
use polkadot_primitives::v1::{BlockData, PoV};
use polkadot_overseer::{Overseer, AllSubsystems};
use polkadot_subsystem::{
Subsystem, SubsystemContext, DummySubsystem,
Subsystem, SubsystemContext, DummySubsystem,
SpawnedSubsystem, FromOverseer,
};
use polkadot_subsystem::messages::{
......@@ -61,13 +61,11 @@ impl Subsystem1 {
let (tx, _) = oneshot::channel();
ctx.send_message(AllMessages::CandidateValidation(
CandidateValidationMessage::Validate(
CandidateValidationMessage::ValidateFromChainState(
Default::default(),
Default::default(),
Default::default(),
PoVBlock {
PoV {
block_data: BlockData(Vec::new()),
},
}.into(),
tx,
)
)).await.unwrap();
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment