Commit 2ee439fe authored by Gav Wood's avatar Gav Wood
Browse files

Merge branch 'master' of github.com:paritytech/polkadot

parents 89265847 ef8a6c7c
Pipeline #28587 passed with stages
in 12 minutes and 58 seconds
......@@ -1873,12 +1873,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "polkadot"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)",
"error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
"polkadot-cli 0.3.0",
"polkadot-cli 0.3.1",
"vergen 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
......@@ -1898,12 +1898,12 @@ dependencies = [
[[package]]
name = "polkadot-cli"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"exit-future 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"polkadot-service 0.3.0",
"polkadot-service 0.3.1",
"structopt 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-cli 0.3.0 (git+https://github.com/paritytech/substrate)",
"tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
......@@ -1916,7 +1916,7 @@ dependencies = [
"futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"parity-codec 2.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"polkadot-cli 0.3.0",
"polkadot-cli 0.3.1",
"polkadot-primitives 0.1.0",
"polkadot-runtime 0.1.0",
"substrate-client 0.1.0 (git+https://github.com/paritytech/substrate)",
......@@ -2051,7 +2051,7 @@ dependencies = [
[[package]]
name = "polkadot-service"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
......
......@@ -100,10 +100,7 @@ impl<C: Collators, P: ProvideRuntimeApi> Future for CollationFetch<C, P>
};
match validate_collation(&*self.client, &self.relay_parent, &x) {
Ok(()) => {
// TODO: generate extrinsic while verifying.
return Ok(Async::Ready((x, Extrinsic)));
}
Ok(e) => return Ok(Async::Ready((x, e))),
Err(e) => {
debug!("Failed to validate parachain due to API error: {}", e);
......@@ -140,12 +137,12 @@ error_chain! {
}
}
/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise.
/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise.
pub fn validate_collation<P>(
client: &P,
relay_parent: &BlockId,
collation: &Collation
) -> Result<(), Error> where
) -> Result<Extrinsic, Error> where
P: ProvideRuntimeApi,
P::Api: ParachainHost<Block>
{
......@@ -167,7 +164,7 @@ pub fn validate_collation<P>(
match parachain::wasm::validate_candidate(&validation_code, params) {
Ok(result) => {
if result.head_data == collation.receipt.head_data.0 {
Ok(())
Ok(Extrinsic)
} else {
Err(ErrorKind::WrongHeadData(
collation.receipt.head_data.0.clone(),
......
......@@ -92,7 +92,7 @@ use dynamic_inclusion::DynamicInclusion;
pub use self::collation::{validate_collation, Collators};
pub use self::error::{ErrorKind, Error};
pub use self::shared_table::{SharedTable, StatementProducer, ProducedStatements, Statement, SignedStatement, GenericStatement};
pub use self::shared_table::{SharedTable, ParachainWork, PrimedParachainWork, Validated, Statement, SignedStatement, GenericStatement};
mod attestation_service;
mod dynamic_inclusion;
......@@ -147,12 +147,8 @@ pub trait Network {
pub struct GroupInfo {
/// Authorities meant to check validity of candidates.
pub validity_guarantors: HashSet<SessionKey>,
/// Authorities meant to check availability of candidate data.
pub availability_guarantors: HashSet<SessionKey>,
/// Number of votes needed for validity.
pub needed_validity: usize,
/// Number of votes needed for availability.
pub needed_availability: usize,
}
/// Sign a table statement against a parent hash.
......@@ -183,15 +179,11 @@ fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId], local_id: Au
bail!(ErrorKind::InvalidDutyRosterLength(authorities.len(), roster.validator_duty.len()))
}
if roster.guarantor_duty.len() != authorities.len() {
bail!(ErrorKind::InvalidDutyRosterLength(authorities.len(), roster.guarantor_duty.len()))
}
let mut local_validation = None;
let mut map = HashMap::new();
let duty_iter = authorities.iter().zip(&roster.validator_duty).zip(&roster.guarantor_duty);
for ((authority, v_duty), a_duty) in duty_iter {
let duty_iter = authorities.iter().zip(&roster.validator_duty);
for (authority, v_duty) in duty_iter {
if authority == &local_id {
local_validation = Some(v_duty.clone());
}
......@@ -204,23 +196,11 @@ fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId], local_id: Au
.insert(authority.clone());
}
}
match *a_duty {
Chain::Relay => {}, // does nothing for now.
Chain::Parachain(ref id) => {
map.entry(id.clone()).or_insert_with(GroupInfo::default)
.availability_guarantors
.insert(authority.clone());
}
}
}
for live_group in map.values_mut() {
let validity_len = live_group.validity_guarantors.len();
let availability_len = live_group.availability_guarantors.len();
live_group.needed_validity = validity_len / 2 + validity_len % 2;
live_group.needed_availability = availability_len / 2 + availability_len % 2;
}
match local_validation {
......@@ -470,8 +450,11 @@ fn dispatch_collation_work<R, C, P>(
});
match res {
Ok(()) =>
router.local_candidate(collation.receipt, collation.block_data, extrinsic),
Ok(()) => {
// TODO: https://github.com/paritytech/polkadot/issues/51
// Erasure-code and provide merkle branches.
router.local_candidate(collation.receipt, collation.block_data, extrinsic)
}
Err(e) =>
warn!(target: "consensus", "Failed to make collation data available: {:?}", e),
}
......
......@@ -22,18 +22,19 @@ use std::sync::Arc;
use extrinsic_store::{Data, Store as ExtrinsicStore};
use table::{self, Table, Context as TableContextTrait};
use polkadot_primitives::{Hash, SessionKey};
use polkadot_primitives::{Block, BlockId, Hash, SessionKey};
use polkadot_primitives::parachain::{
Id as ParaId, BlockData, Collation, Extrinsic, CandidateReceipt,
AttestedCandidate,
AttestedCandidate, ParachainHost
};
use parking_lot::Mutex;
use futures::{future, prelude::*};
use futures::prelude::*;
use super::{GroupInfo, TableRouter};
use self::includable::IncludabilitySender;
use primitives::ed25519;
use runtime_primitives::{traits::ProvideRuntimeApi};
mod includable;
......@@ -52,15 +53,8 @@ impl table::Context for TableContext {
self.groups.get(group).map_or(false, |g| g.validity_guarantors.contains(authority))
}
fn is_availability_guarantor_of(&self, authority: &SessionKey, group: &ParaId) -> bool {
self.groups.get(group).map_or(false, |g| g.availability_guarantors.contains(authority))
}
fn requisite_votes(&self, group: &ParaId) -> (usize, usize) {
self.groups.get(group).map_or(
(usize::max_value(), usize::max_value()),
|g| (g.needed_validity, g.needed_availability),
)
fn requisite_votes(&self, group: &ParaId) -> usize {
self.groups.get(group).map_or(usize::max_value(), |g| g.needed_validity)
}
}
......@@ -85,7 +79,6 @@ struct SharedTableInner {
table: Table<TableContext>,
proposed_digest: Option<Hash>,
checked_validity: HashSet<Hash>,
checked_availability: HashSet<Hash>,
trackers: Vec<IncludabilitySender>,
extrinsic_store: ExtrinsicStore,
}
......@@ -101,9 +94,8 @@ impl SharedTableInner {
context: &TableContext,
router: &R,
statement: table::SignedStatement,
) -> Option<StatementProducer<
) -> Option<ParachainWork<
<R::FetchCandidate as IntoFuture>::Future,
<R::FetchExtrinsic as IntoFuture>::Future,
>> {
let summary = match self.table.import_statement(context, statement) {
Some(summary) => summary,
......@@ -114,42 +106,25 @@ impl SharedTableInner {
let local_id = context.local_id();
let is_validity_member = context.is_member_of(&local_id, &summary.group_id);
let is_availability_member =
context.is_availability_guarantor_of(&local_id, &summary.group_id);
let para_member = context.is_member_of(&local_id, &summary.group_id);
let digest = &summary.candidate;
// TODO: consider a strategy based on the number of candidate votes as well.
// only check validity if this wasn't locally proposed.
let checking_validity = is_validity_member
let extra_work = para_member
&& self.proposed_digest.as_ref().map_or(true, |d| d != digest)
&& self.checked_validity.insert(digest.clone());
let checking_availability = is_availability_member
&& self.checked_availability.insert(digest.clone());
let work = if checking_validity || checking_availability {
let work = if extra_work {
match self.table.get_candidate(&digest) {
None => None, // TODO: handle table inconsistency somehow?
Some(candidate) => {
let fetch_block_data =
router.fetch_block_data(candidate).into_future().fuse();
let fetch_extrinsic = if checking_availability {
Some(
router.fetch_extrinsic_data(candidate).into_future().fuse()
)
} else {
None
};
let fetch_block_data = router.fetch_block_data(candidate).into_future();
Some(Work {
candidate_receipt: candidate.clone(),
fetch_block_data,
fetch_extrinsic,
evaluate: checking_validity,
ensure_available: checking_availability,
})
}
}
......@@ -157,8 +132,7 @@ impl SharedTableInner {
None
};
work.map(|work| StatementProducer {
produced_statements: Default::default(),
work.map(|work| ParachainWork {
extrinsic_store: self.extrinsic_store.clone(),
relay_parent: context.parent_hash.clone(),
work
......@@ -175,130 +149,114 @@ impl SharedTableInner {
}
}
/// Produced statements about a specific candidate.
/// Both may be `None`.
#[derive(Default)]
pub struct ProducedStatements {
/// Produced after validating a candidate.
pub struct Validated {
/// A statement about the validity of the candidate.
pub validity: Option<table::Statement>,
/// A statement about availability of data. If this is `Some`,
/// then `block_data` and `extrinsic` should be `Some` as well.
pub availability: Option<table::Statement>,
pub validity: table::Statement,
/// Block data to ensure availability of.
pub block_data: Option<BlockData>,
pub block_data: BlockData,
/// Extrinsic data to ensure availability of.
pub extrinsic: Option<Extrinsic>,
pub extrinsic: Extrinsic,
}
/// Future that produces statements about a specific candidate.
pub struct StatementProducer<D: Future, E: Future> {
produced_statements: ProducedStatements,
work: Work<D, E>,
/// Future that performs parachain validation work.
pub struct ParachainWork<D: Future> {
work: Work<D>,
relay_parent: Hash,
extrinsic_store: ExtrinsicStore,
}
impl<D: Future, E: Future> StatementProducer<D, E> {
/// Attach a function for verifying fetched collation to the statement producer.
/// This will transform it into a future.
///
/// The collation-checking function should return `true` if known to be valid,
/// `false` if known to be invalid, and `None` if unable to determine.
pub fn prime<C: FnMut(Collation) -> Option<bool>>(self, check_candidate: C) -> PrimedStatementProducer<D, E, C> {
PrimedStatementProducer {
inner: self,
check_candidate,
}
impl<D: Future> ParachainWork<D> {
/// Prime the parachain work with an API reference for extracting
/// chain information.
pub fn prime<P: ProvideRuntimeApi>(self, api: Arc<P>)
-> PrimedParachainWork<
D,
impl Send + FnMut(&BlockId, &Collation) -> bool,
>
where
P: Send + Sync + 'static,
P::Api: ParachainHost<Block>,
{
let validate = move |id: &_, collation: &_| {
let res = ::collation::validate_collation(
&*api,
id,
collation,
);
match res {
Ok(_) => true,
Err(e) => {
debug!(target: "consensus", "Encountered bad collation: {}", e);
false
}
}
};
PrimedParachainWork { inner: self, validate }
}
/// Prime the parachain work with a custom validation function.
pub fn prime_with<F>(self, validate: F) -> PrimedParachainWork<D, F>
where F: FnMut(&BlockId, &Collation) -> bool
{
PrimedParachainWork { inner: self, validate }
}
}
struct Work<D: Future, E: Future> {
struct Work<D: Future> {
candidate_receipt: CandidateReceipt,
fetch_block_data: future::Fuse<D>,
fetch_extrinsic: Option<future::Fuse<E>>,
evaluate: bool,
ensure_available: bool,
fetch_block_data: D,
}
/// Primed statement producer.
pub struct PrimedStatementProducer<D: Future, E: Future, C> {
inner: StatementProducer<D, E>,
check_candidate: C,
pub struct PrimedParachainWork<D: Future, F> {
inner: ParachainWork<D>,
validate: F,
}
impl<D, E, C, Err> Future for PrimedStatementProducer<D, E, C>
impl<D, F, Err> Future for PrimedParachainWork<D, F>
where
D: Future<Item=BlockData,Error=Err>,
E: Future<Item=Extrinsic,Error=Err>,
C: FnMut(Collation) -> Option<bool>,
F: FnMut(&BlockId, &Collation) -> bool,
Err: From<::std::io::Error>,
{
type Item = ProducedStatements;
type Item = Validated;
type Error = Err;
fn poll(&mut self) -> Poll<ProducedStatements, Err> {
fn poll(&mut self) -> Poll<Validated, Err> {
let work = &mut self.inner.work;
let candidate = &work.candidate_receipt;
let statements = &mut self.inner.produced_statements;
let mut candidate_hash = None;
let mut candidate_hash = move ||
candidate_hash.get_or_insert_with(|| candidate.hash()).clone();
if let Async::Ready(block_data) = work.fetch_block_data.poll()? {
statements.block_data = Some(block_data.clone());
if work.evaluate {
let is_good = (self.check_candidate)(Collation {
block_data,
receipt: work.candidate_receipt.clone(),
});
let hash = candidate_hash();
debug!(target: "consensus", "Making validity statement about candidate {}: is_good? {:?}", hash, is_good);
statements.validity = match is_good {
Some(true) => Some(GenericStatement::Valid(hash)),
Some(false) => Some(GenericStatement::Invalid(hash)),
None => None,
};
work.evaluate = false;
}
}
if let Async::Ready(Some(extrinsic)) = work.fetch_extrinsic.poll()? {
if work.ensure_available {
let hash = candidate_hash();
debug!(target: "consensus", "Claiming candidate {} available.", hash);
statements.extrinsic = Some(extrinsic);
statements.availability = Some(GenericStatement::Available(hash));
let block = try_ready!(work.fetch_block_data.poll());
let is_good = (self.validate)(
&BlockId::hash(self.inner.relay_parent),
&Collation { block_data: block.clone(), receipt: candidate.clone() },
);
work.ensure_available = false;
}
}
let candidate_hash = candidate.hash();
let done = match (work.evaluate, work.ensure_available) {
(false, false) => true,
_ => false,
debug!(target: "consensus", "Making validity statement about candidate {}: is_good? {:?}", candidate_hash, is_good);
let validity_statement = match is_good {
true => GenericStatement::Valid(candidate_hash),
false => GenericStatement::Invalid(candidate_hash),
};
if done {
// commit claimed-available data to disk before returning statements from the future.
if let (&Some(ref block), extrinsic) = (&statements.block_data, &statements.extrinsic) {
self.inner.extrinsic_store.make_available(Data {
relay_parent: self.inner.relay_parent,
parachain_id: work.candidate_receipt.parachain_index,
candidate_hash: candidate_hash(),
block_data: block.clone(),
extrinsic: extrinsic.clone(),
})?;
}
Ok(Async::Ready(::std::mem::replace(statements, Default::default())))
} else {
Ok(Async::NotReady)
}
let extrinsic = Extrinsic;
self.inner.extrinsic_store.make_available(Data {
relay_parent: self.inner.relay_parent,
parachain_id: work.candidate_receipt.parachain_index,
candidate_hash,
block_data: block.clone(),
extrinsic: Some(extrinsic.clone()),
})?;
Ok(Async::Ready(Validated {
validity: validity_statement,
block_data: block,
extrinsic,
}))
}
}
......@@ -334,7 +292,6 @@ impl SharedTable {
table: Table::default(),
proposed_digest: None,
checked_validity: HashSet::new(),
checked_availability: HashSet::new(),
trackers: Vec::new(),
extrinsic_store,
}))
......@@ -364,9 +321,8 @@ impl SharedTable {
&self,
router: &R,
statement: table::SignedStatement,
) -> Option<StatementProducer<
) -> Option<ParachainWork<
<R::FetchCandidate as IntoFuture>::Future,
<R::FetchExtrinsic as IntoFuture>::Future,
>> {
self.inner.lock().import_remote_statement(&*self.context, router, statement)
}
......@@ -381,9 +337,8 @@ impl SharedTable {
where
R: TableRouter,
I: IntoIterator<Item=table::SignedStatement>,
U: ::std::iter::FromIterator<Option<StatementProducer<
U: ::std::iter::FromIterator<Option<ParachainWork<
<R::FetchCandidate as IntoFuture>::Future,
<R::FetchExtrinsic as IntoFuture>::Future,
>>>,
{
let mut inner = self.inner.lock();
......@@ -394,25 +349,12 @@ impl SharedTable {
}
/// Sign and import a local statement.
///
/// For candidate statements, this may also produce a second signed statement
/// concerning the availability of the candidate data.
pub fn sign_and_import(&self, statement: table::Statement)
-> (SignedStatement, Option<SignedStatement>)
-> SignedStatement
{
let (proposed_digest, availability) = match statement {
GenericStatement::Candidate(ref c) => {
let mut availability = None;
let hash = c.hash();
// TODO: actually store the data in an availability store of some kind.
if self.context.is_availability_guarantor_of(&self.context.local_id(), &c.parachain_index) {
availability = Some(self.context.sign_statement(GenericStatement::Available(hash)));
}
(Some(hash), availability)
}
_ => (None, None),
let proposed_digest = match statement {
GenericStatement::Candidate(ref c) => Some(c.hash()),
_ => None,
};
let signed_statement = self.context.sign_statement(statement);
......@@ -424,12 +366,7 @@ impl SharedTable {
inner.table.import_statement(&*self.context, signed_statement.clone());
// ensure the availability statement is imported after the candidate.
if let Some(a) = availability.clone() {
inner.table.import_statement(&*self.context, a);
}
(signed_statement, availability)
signed_statement
}
/// Execute a closure using a specific candidate.
......@@ -454,7 +391,6 @@ impl SharedTable {
table_attestations.into_iter()
.map(|attested| AttestedCandidate {
candidate: attested.candidate,
availability_votes: attested.availability_votes,
validity_votes: attested.validity_votes.into_iter().map(|(a, v)| match v {
GAttestation::Implicit(s) => (a, ValidityAttestation::Implicit(s)),
GAttestation::Explicit(s) => (a, ValidityAttestation::Explicit(s)),
......@@ -468,7 +404,7 @@ impl SharedTable {
self.group_info().len()
}
/// Get the number of parachains which have available candidates.
/// Get the number of parachains whose candidates may be included.
pub fn includable_count(&self) -> usize {
self.inner.lock().table.includable_count()
}
......@@ -501,22 +437,23 @@ impl SharedTable {
mod tests {
use super::*;
use substrate_keyring::Keyring;
use futures::future;
#[derive(Clone)]
struct DummyRouter;
impl TableRouter for DummyRouter {
type Error = ::std::io::Error;
type FetchCandidate = ::futures::future::Empty<BlockData,Self::Error>;
type FetchExtrinsic = ::futures::future::Empty<Extrinsic,Self::Error>;
type FetchCandidate = ::futures::future::FutureResult<BlockData,Self::Error>;
type FetchExtrinsic = ::futures::future::FutureResult<Extrinsic,Self::Error>;
fn local_candidate(&self, _candidate: CandidateReceipt, _block_data: BlockData, _extrinsic: Extrinsic) {
}
fn fetch_block_data(&self, _candidate: &CandidateReceipt) -> Self::FetchCandidate {
::futures::future::empty()
future::ok(BlockData(vec![1, 2, 3, 4, 5]))
}
fn fetch_extrinsic_data(&self, _candidate: &CandidateReceipt) -> Self::FetchExtrinsic {
::futures::future::empty()
future::ok(Extrinsic)
}
}
......@@ -534,9 +471,7 @@ mod tests {
groups.insert(para_id, GroupInfo {
validity_guarantors: [local_id, validity_other].iter().cloned().collect(),
availability_guarantors: Default::default(),
needed_validity: 2,
needed_availability: 0,
});
let shared_table = SharedTable::new(
......@@ -566,17 +501,14 @@ mod tests {
sender: validity_other,
};