Commit f41a4959 authored by asynchronous rob's avatar asynchronous rob Committed by GitHub
Browse files

Minimal parachain framework part 1 (#113)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* grumbles

* update WASM checkins

* remove dependency on tokio-timer
parent 1d44c24d
......@@ -5,7 +5,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
[dependencies]
error-chain = "0.11"
log = "0.3"
polkadot-executor = { path = "../executor" }
polkadot-runtime = { path = "../runtime" }
polkadot-primitives = { path = "../primitives" }
......
......@@ -40,8 +40,8 @@ use polkadot_executor::Executor as LocalDispatch;
use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
use state_machine::OverlayedChanges;
use primitives::{AccountId, BlockId, Hash, Index, SessionKey, Timestamp};
use primitives::parachain::DutyRoster;
use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall};
use primitives::parachain::{DutyRoster, CandidateReceipt, Id as ParaId};
use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall, ParachainsCall};
error_chain! {
errors {
......@@ -135,12 +135,21 @@ pub trait PolkadotApi {
/// Get the index of an account at a block.
fn index(&self, at: &Self::CheckedBlockId, account: AccountId) -> Result<Index>;
/// Get the active parachains at a block.
fn active_parachains(&self, at: &Self::CheckedBlockId) -> Result<Vec<ParaId>>;
/// Evaluate a block and see if it gives an error.
fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<()>;
/// Get the validation code of a parachain at a block. If the parachain is active, this will always return `Some`.
fn parachain_code(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
/// Get the chain head of a parachain. If the parachain is active, this will always return `Some`.
fn parachain_head(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
/// Evaluate a block. Returns true if the block is good, false if it is known to be bad,
/// and an error if we can't evaluate for some reason.
fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<bool>;
/// Create a block builder on top of the parent block.
fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp) -> Result<Self::BlockBuilder>;
fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder>;
}
/// A checked block ID used for the substrate-client implementation of CheckedBlockId;
......@@ -213,15 +222,36 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
with_runtime!(self, at, ::runtime::Timestamp::now)
}
fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<()> {
with_runtime!(self, at, || ::runtime::Executive::execute_block(block))
fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<bool> {
use substrate_executor::error::ErrorKind as ExecErrorKind;
let res = with_runtime!(self, at, || ::runtime::Executive::execute_block(block));
match res {
Ok(()) => Ok(true),
Err(err) => match err.kind() {
&ErrorKind::Executor(ExecErrorKind::Runtime) => Ok(false),
_ => Err(err)
}
}
}
fn index(&self, at: &CheckedId, account: AccountId) -> Result<Index> {
with_runtime!(self, at, || ::runtime::System::account_index(account))
}
fn build_block(&self, parent: &CheckedId, timestamp: Timestamp) -> Result<Self::BlockBuilder> {
fn active_parachains(&self, at: &CheckedId) -> Result<Vec<ParaId>> {
with_runtime!(self, at, ::runtime::Parachains::active_parachains)
}
fn parachain_code(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
with_runtime!(self, at, || ::runtime::Parachains::parachain_code(parachain))
}
fn parachain_head(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
with_runtime!(self, at, || ::runtime::Parachains::parachain_head(parachain))
}
fn build_block(&self, parent: &CheckedId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder> {
let parent = parent.block_id();
let header = Header {
parent_hash: self.block_hash_from_id(parent)?.ok_or(ErrorKind::UnknownBlock(*parent))?,
......@@ -239,6 +269,14 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
function: Call::Timestamp(TimestampCall::set(timestamp)),
},
signature: Default::default(),
},
UncheckedExtrinsic {
extrinsic: Extrinsic {
signed: Default::default(),
index: Default::default(),
function: Call::Parachains(ParachainsCall::set_heads(parachains)),
},
signature: Default::default(),
}
];
......@@ -275,7 +313,7 @@ pub struct ClientBlockBuilder<S> {
impl<S: state_machine::Backend> ClientBlockBuilder<S>
where S::Error: Into<client::error::Error>
{
// initialises a block ready to allow extrinsics to be applied.
// initialises a block, ready to allow extrinsics to be applied.
fn initialise_block(&mut self) -> Result<()> {
let result = {
let mut ext = state_machine::Ext::new(&mut self.changes, &self.state);
......@@ -406,7 +444,7 @@ mod tests {
let client = client();
let id = client.check_id(BlockId::Number(0)).unwrap();
let block_builder = client.build_block(&id, 1_000_000).unwrap();
let block_builder = client.build_block(&id, 1_000_000, Vec::new()).unwrap();
let block = block_builder.bake();
assert_eq!(block.header.number, 1);
......
[package]
name = "polkadot-collator"
version = "0.1.0"
authors = ["Parity Technologies <rphmeier@gmail.com>"]
authors = ["Parity Technologies <admin@parity.io>"]
description = "Abstract collation logic"
[dependencies]
futures = "0.1.17"
substrate-codec = { path = "../../substrate/codec", version = "0.1" }
substrate-primitives = { path = "../../substrate/primitives", version = "0.1" }
polkadot-primitives = { path = "../primitives", version = "0.1" }
polkadot-parachain = { path = "../parachain", version = "0.1" }
......@@ -45,6 +45,7 @@
//! to be performed, as the collation logic itself.
extern crate futures;
extern crate substrate_codec as codec;
extern crate substrate_primitives as primitives;
extern crate polkadot_primitives;
......@@ -82,7 +83,6 @@ pub trait RelayChainContext {
}
/// Collate the necessary ingress queue using the given context.
// TODO: impl trait
pub fn collate_ingress<'a, R>(relay_context: R)
-> Box<Future<Item=ConsolidatedIngress, Error=R::Error> + 'a>
where
......@@ -105,7 +105,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
// and then by the parachain ID.
//
// then transform that into the consolidated egress queue.
let future = stream::futures_unordered(egress_fetch)
Box::new(stream::futures_unordered(egress_fetch)
.fold(BTreeMap::new(), |mut map, (routing_id, egresses)| {
for (depth, egress) in egresses.into_iter().rev().enumerate() {
let depth = -(depth as i64);
......@@ -116,9 +116,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
})
.map(|ordered| ordered.into_iter().map(|((_, id), egress)| (id, egress)))
.map(|i| i.collect::<Vec<_>>())
.map(ConsolidatedIngress);
Box::new(future)
.map(ConsolidatedIngress))
}
/// Produce a candidate for the parachain.
......
......@@ -13,6 +13,7 @@ log = "0.3"
exit-future = "0.1"
polkadot-api = { path = "../api" }
polkadot-collator = { path = "../collator" }
polkadot-parachain = { path = "../parachain" }
polkadot-primitives = { path = "../primitives" }
polkadot-runtime = { path = "../runtime" }
polkadot-statement-table = { path = "../statement-table" }
......
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Validator-side view of collation.
//!
//! This module contains type definitions, a trait for a batch of collators, and a trait for
//! attempting to fetch a collation repeatedly until a valid one is obtained.
use std::sync::Arc;
use polkadot_api::PolkadotApi;
use polkadot_primitives::{Hash, AccountId};
use polkadot_primitives::parachain::{Id as ParaId, Chain, BlockData, Extrinsic, CandidateReceipt};
use futures::prelude::*;
/// A full collation.
pub struct Collation {
/// Block data.
pub block_data: BlockData,
/// The candidate receipt itself.
pub receipt: CandidateReceipt,
}
/// Encapsulates connections to collators and allows collation on any parachain.
///
/// This is expected to be a lightweight, shared type like an `Arc`.
pub trait Collators: Clone {
/// Errors when producing collations.
type Error;
/// A full collation.
type Collation: IntoFuture<Item=Collation,Error=Self::Error>;
/// Collate on a specific parachain, building on a given relay chain parent hash.
fn collate(&self, parachain: ParaId, relay_parent: Hash) -> Self::Collation;
/// Note a bad collator. TODO: take proof
fn note_bad_collator(&self, collator: AccountId);
}
/// A future which resolves when a collation is available.
///
/// This future is fused.
pub struct CollationFetch<C: Collators, P: PolkadotApi> {
parachain: Option<ParaId>,
relay_parent_hash: Hash,
relay_parent: P::CheckedBlockId,
collators: C,
live_fetch: Option<<C::Collation as IntoFuture>::Future>,
client: Arc<P>,
}
impl<C: Collators, P: PolkadotApi> CollationFetch<C, P> {
/// Create a new collation fetcher for the given chain.
pub fn new(parachain: Chain, relay_parent: P::CheckedBlockId, relay_parent_hash: Hash, collators: C, client: Arc<P>) -> Self {
CollationFetch {
relay_parent_hash,
relay_parent,
collators,
client,
parachain: match parachain {
Chain::Parachain(id) => Some(id),
Chain::Relay => None,
},
live_fetch: None,
}
}
}
impl<C: Collators, P: PolkadotApi> Future for CollationFetch<C, P> {
type Item = (Collation, Extrinsic);
type Error = C::Error;
fn poll(&mut self) -> Poll<(Collation, Extrinsic), C::Error> {
let parachain = match self.parachain.as_ref() {
Some(p) => p.clone(),
None => return Ok(Async::NotReady),
};
loop {
let x = {
let (r, c) = (self.relay_parent_hash, &self.collators);
let poll = self.live_fetch
.get_or_insert_with(move || c.collate(parachain, r).into_future())
.poll();
if let Err(_) = poll { self.parachain = None }
try_ready!(poll)
};
match validate_collation(&*self.client, &self.relay_parent, &x) {
Ok(()) => {
self.parachain = None;
// TODO: generate extrinsic while verifying.
return Ok(Async::Ready((x, Extrinsic)));
}
Err(e) => {
debug!("Failed to validate parachain due to API error: {}", e);
// just continue if we got a bad collation or failed to validate
self.live_fetch = None;
self.collators.note_bad_collator(x.receipt.collator)
}
}
}
}
}
// Errors that can occur when validating a parachain.
error_chain! {
types { Error, ErrorKind, ResultExt; }
errors {
InactiveParachain(id: ParaId) {
description("Collated for inactive parachain"),
display("Collated for inactive parachain: {:?}", id),
}
ValidationFailure {
description("Parachain candidate failed validation."),
display("Parachain candidate failed validation."),
}
WrongHeadData(expected: Vec<u8>, got: Vec<u8>) {
description("Parachain validation produced wrong head data."),
display("Parachain validation produced wrong head data (expected: {:?}, got {:?}", expected, got),
}
}
links {
PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
}
}
/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise.
pub fn validate_collation<P: PolkadotApi>(client: &P, relay_parent: &P::CheckedBlockId, collation: &Collation) -> Result<(), Error> {
use parachain::{self, ValidationParams};
let para_id = collation.receipt.parachain_index;
let validation_code = client.parachain_code(relay_parent, para_id)?
.ok_or_else(|| ErrorKind::InactiveParachain(para_id))?;
let chain_head = client.parachain_head(relay_parent, para_id)?
.ok_or_else(|| ErrorKind::InactiveParachain(para_id))?;
let params = ValidationParams {
parent_head: chain_head,
block_data: collation.block_data.0.clone(),
};
match parachain::wasm::validate_candidate(&validation_code, params) {
Ok(result) => {
if result.head_data == collation.receipt.head_data.0 {
Ok(())
} else {
Err(ErrorKind::WrongHeadData(
collation.receipt.head_data.0.clone(),
result.head_data
).into())
}
}
Err(_) => Err(ErrorKind::ValidationFailure.into())
}
}
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Dynamic inclusion threshold over time.
use std::time::{Duration, Instant};
fn duration_to_micros(duration: &Duration) -> u64 {
duration.as_secs() * 1_000_000 + (duration.subsec_nanos() / 1000) as u64
}
/// Dynamic inclusion threshold over time.
///
/// The acceptable proportion of parachains which must have parachain candidates
/// reduces over time (eventually going to zero).
#[derive(Debug, Clone)]
pub struct DynamicInclusion {
start: Instant,
y: u64,
m: u64,
}
impl DynamicInclusion {
/// Constructs a new dynamic inclusion threshold calculator based on the time now,
/// how many parachain candidates are required at the beginning, and when an empty
/// block will be allowed.
pub fn new(initial: usize, start: Instant, allow_empty: Duration) -> Self {
// linear function f(n_candidates) -> valid after microseconds
// f(0) = allow_empty
// f(initial) = 0
// m is actually the negative slope to avoid using signed arithmetic.
let (y, m) = if initial != 0 {
let y = duration_to_micros(&allow_empty);
(y, y / initial as u64)
} else {
(0, 0)
};
DynamicInclusion {
start,
y,
m,
}
}
/// Returns the duration from `now` after which the amount of included parachain candidates
/// would be enough, or `None` if it is sufficient now.
///
/// Panics if `now` is earlier than the `start`.
pub fn acceptable_in(&self, now: Instant, included: usize) -> Option<Duration> {
let elapsed = now.duration_since(self.start);
let elapsed = duration_to_micros(&elapsed);
let valid_after = self.y.saturating_sub(self.m * included as u64);
if elapsed >= valid_after {
None
} else {
Some(Duration::from_millis((valid_after - elapsed) as u64 / 1000))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn full_immediately_allowed() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
10,
now,
Duration::from_millis(4000),
);
assert!(dynamic.acceptable_in(now, 10).is_none());
assert!(dynamic.acceptable_in(now, 11).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 10).is_none());
}
#[test]
fn half_allowed_halfway() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
10,
now,
Duration::from_millis(4000),
);
assert_eq!(dynamic.acceptable_in(now, 5), Some(Duration::from_millis(2000)));
assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 5).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(3000), 5).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(4000), 5).is_none());
}
#[test]
fn zero_initial_is_flat() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
0,
now,
Duration::from_secs(10_000),
);
for i in 0..10_001 {
let now = now + Duration::from_secs(i);
assert!(dynamic.acceptable_in(now, 0).is_none());
assert!(dynamic.acceptable_in(now, 1).is_none());
assert!(dynamic.acceptable_in(now, 10).is_none());
}
}
}
......@@ -16,45 +16,30 @@
//! Errors that can occur during the consensus process.
use primitives::block::{HeaderHash, Number};
use polkadot_primitives::AccountId;
error_chain! {
links {
PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
Bft(::bft::Error, ::bft::ErrorKind);
}
foreign_links {
Io(::std::io::Error);
SharedIo(::futures::future::SharedError<::std::io::Error>);
}
errors {
InvalidDutyRosterLength(expected: usize, got: usize) {
description("Duty Roster had invalid length"),
display("Invalid duty roster length: expected {}, got {}", expected, got),
}
ProposalNotForPolkadot {
description("Proposal provided not a Polkadot block."),
display("Proposal provided not a Polkadot block."),
}
TimestampInFuture {
description("Proposal had timestamp too far in the future."),
display("Proposal had timestamp too far in the future."),
}
WrongParentHash(expected: HeaderHash, got: HeaderHash) {
description("Proposal had wrong parent hash."),
display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got),
NotValidator(id: AccountId) {
description("Local account ID not a validator at this block."),
display("Local account ID ({:?}) not a validator at this block.", id),
}
WrongNumber(expected: Number, got: Number) {
description("Proposal had wrong number."),
display("Proposal had wrong number. Expected {:?}, got {:?}", expected, got),
PrematureDestruction {
description("Proposer destroyed before finishing proposing or evaluating"),
display("Proposer destroyed before finishing proposing or evaluating"),
}
ProposalTooLarge(size: usize) {
description("Proposal exceeded the maximum size."),
display(
"Proposal exceeded the maximum size of {} by {} bytes.",
::MAX_TRANSACTIONS_SIZE, ::MAX_TRANSACTIONS_SIZE.saturating_sub(*size)
),
Timer(e: String) {
description("Failed to register or resolve async timer."),
display("Timer failed: {}", e),
}
Executor(e: ::futures::future::ExecuteErrorKind) {
description("Unable to dispatch agreement future"),
......
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Polkadot block evaluation and evaluation errors.
use super::MAX_TRANSACTIONS_SIZE;
use codec::Slicable;
use polkadot_runtime::Block as PolkadotGenericBlock;
use polkadot_primitives::Timestamp;
use polkadot_primitives::parachain::Id as ParaId;
use primitives::block::{Block as SubstrateBlock, HeaderHash, Number as BlockNumber};
use transaction_pool::PolkadotBlock;
error_chain! {
links {
PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
}
errors {
ProposalNotForPolkadot {
description("Proposal provided not a Polkadot block."),
display("Proposal provided not a Polkadot block."),
}
TimestampInFuture {
description("Proposal had timestamp too far in the future."),
display("Proposal had timestamp too far in the future."),
}
TooManyCandidates(expected: usize, got: usize) {
description("Proposal included more candidates than is possible."),
display("Proposal included {} candidates for {} parachains", got, expected),
}
ParachainOutOfOrder {
description("Proposal included parachains out of order."),
display("Proposal included parachains out of order."),
}
UnknownParachain(id: ParaId) {
description("Proposal included unregistered parachain."),
display("Proposal included unregistered parachain {:?}", id),
}
WrongParentHash(expected: HeaderHash, got: HeaderHash) {
description("Proposal had wrong parent hash."),
display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got),
}
WrongNumber(expected: BlockNumber, got: BlockNumber) {
description("Proposal had wrong number."),
display("Proposal had wrong number. Expected {:?}, got {:?}", expected, got),
}
ProposalTooLarge(size: usize) {
description("Proposal exceeded the maximum size."),
display(
"Proposal exceeded the maximum size of {} by {} bytes.",