Unverified Commit ed2c4cab authored by asynchronous rob's avatar asynchronous rob Committed by GitHub
Browse files

Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue
parent aea08acf
Pipeline #86682 passed with stages
in 22 minutes and 5 seconds
...@@ -10,34 +10,6 @@ dependencies = [ ...@@ -10,34 +10,6 @@ dependencies = [
"regex", "regex",
] ]
[[package]]
name = "adder"
version = "0.7.29-pre1"
dependencies = [
"dlmalloc",
"parity-scale-codec",
"polkadot-parachain",
"sp-io",
"substrate-wasm-builder-runner 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.5.0",
]
[[package]]
name = "adder-collator"
version = "0.1.0"
dependencies = [
"adder",
"futures 0.3.4",
"parity-scale-codec",
"parking_lot 0.10.0",
"polkadot-collator",
"polkadot-parachain",
"polkadot-primitives",
"sc-client",
"sc-client-api",
"sp-core",
]
[[package]] [[package]]
name = "adler32" name = "adler32"
version = "1.0.4" version = "1.0.4"
...@@ -1778,13 +1750,6 @@ dependencies = [ ...@@ -1778,13 +1750,6 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "halt"
version = "0.7.29-pre1"
dependencies = [
"substrate-wasm-builder-runner 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "hash-db" name = "hash-db"
version = "0.15.2" version = "0.15.2"
...@@ -4294,9 +4259,7 @@ dependencies = [ ...@@ -4294,9 +4259,7 @@ dependencies = [
name = "polkadot-parachain" name = "polkadot-parachain"
version = "0.7.29-pre1" version = "0.7.29-pre1"
dependencies = [ dependencies = [
"adder",
"derive_more 0.99.3", "derive_more 0.99.3",
"halt",
"log 0.4.8", "log 0.4.8",
"parity-scale-codec", "parity-scale-codec",
"parking_lot 0.10.0", "parking_lot 0.10.0",
...@@ -4309,7 +4272,6 @@ dependencies = [ ...@@ -4309,7 +4272,6 @@ dependencies = [
"sp-runtime-interface", "sp-runtime-interface",
"sp-std", "sp-std",
"sp-wasm-interface", "sp-wasm-interface",
"tiny-keccak 1.5.0",
] ]
[[package]] [[package]]
...@@ -4317,7 +4279,6 @@ name = "polkadot-primitives" ...@@ -4317,7 +4279,6 @@ name = "polkadot-primitives"
version = "0.7.29-pre1" version = "0.7.29-pre1"
dependencies = [ dependencies = [
"bitvec", "bitvec",
"pallet-babe",
"parity-scale-codec", "parity-scale-codec",
"polkadot-parachain", "polkadot-parachain",
"pretty_assertions", "pretty_assertions",
...@@ -7524,6 +7485,65 @@ dependencies = [ ...@@ -7524,6 +7485,65 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "test-parachain-adder"
version = "0.7.29-pre1"
dependencies = [
"dlmalloc",
"parity-scale-codec",
"polkadot-parachain",
"sp-io",
"substrate-wasm-builder-runner 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.5.0",
]
[[package]]
name = "test-parachain-adder-collator"
version = "0.1.0"
dependencies = [
"futures 0.3.4",
"parity-scale-codec",
"parking_lot 0.10.0",
"polkadot-collator",
"polkadot-parachain",
"polkadot-primitives",
"sc-client",
"sc-client-api",
"sp-core",
"test-parachain-adder",
]
[[package]]
name = "test-parachain-code-upgrader"
version = "0.7.22"
dependencies = [
"dlmalloc",
"parity-scale-codec",
"polkadot-parachain",
"sp-io",
"substrate-wasm-builder-runner 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"tiny-keccak 1.5.0",
]
[[package]]
name = "test-parachain-halt"
version = "0.7.29-pre1"
dependencies = [
"substrate-wasm-builder-runner 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "test-parachains"
version = "0.7.22"
dependencies = [
"parity-scale-codec",
"polkadot-parachain",
"test-parachain-adder",
"test-parachain-code-upgrader",
"test-parachain-halt",
"tiny-keccak 1.5.0",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.11.0" version = "0.11.0"
......
...@@ -41,13 +41,15 @@ members = [ ...@@ -41,13 +41,15 @@ members = [
"service", "service",
"validation", "validation",
"test-parachains/adder", "parachain/test-parachains",
"test-parachains/adder/collator", "parachain/test-parachains/adder",
"parachain/test-parachains/adder/collator",
"parachain/test-parachains/code-upgrader",
] ]
exclude = [ exclude = [
"runtime/polkadot/wasm", "runtime/polkadot/wasm",
"runtime/kusama/wasm", "runtime/kusama/wasm",
"test-parachains/adder/wasm", "parachain/test-parachains/adder/wasm",
] ]
[badges] [badges]
......
...@@ -59,7 +59,7 @@ use polkadot_primitives::{ ...@@ -59,7 +59,7 @@ use polkadot_primitives::{
BlockId, Hash, Block, BlockId, Hash, Block,
parachain::{ parachain::{
self, BlockData, DutyRoster, HeadData, Id as ParaId, self, BlockData, DutyRoster, HeadData, Id as ParaId,
PoVBlock, ValidatorId, CollatorPair, LocalValidationData PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
} }
}; };
use polkadot_cli::{ use polkadot_cli::{
...@@ -154,7 +154,8 @@ pub trait ParachainContext: Clone { ...@@ -154,7 +154,8 @@ pub trait ParachainContext: Clone {
fn produce_candidate( fn produce_candidate(
&mut self, &mut self,
relay_parent: Hash, relay_parent: Hash,
status: LocalValidationData, global_validation: GlobalValidationSchedule,
local_validation: LocalValidationData,
) -> Self::ProduceCandidate; ) -> Self::ProduceCandidate;
} }
...@@ -162,6 +163,7 @@ pub trait ParachainContext: Clone { ...@@ -162,6 +163,7 @@ pub trait ParachainContext: Clone {
pub async fn collate<P>( pub async fn collate<P>(
relay_parent: Hash, relay_parent: Hash,
local_id: ParaId, local_id: ParaId,
global_validation: GlobalValidationSchedule,
local_validation_data: LocalValidationData, local_validation_data: LocalValidationData,
mut para_context: P, mut para_context: P,
key: Arc<CollatorPair>, key: Arc<CollatorPair>,
...@@ -173,6 +175,7 @@ pub async fn collate<P>( ...@@ -173,6 +175,7 @@ pub async fn collate<P>(
{ {
let (block_data, head_data) = para_context.produce_candidate( let (block_data, head_data) = para_context.produce_candidate(
relay_parent, relay_parent,
global_validation,
local_validation_data, local_validation_data,
).map_err(Error::Collator).await?; ).map_err(Error::Collator).await?;
...@@ -281,6 +284,7 @@ fn build_collator_service<S, P, Extrinsic>( ...@@ -281,6 +284,7 @@ fn build_collator_service<S, P, Extrinsic>(
let work = future::lazy(move |_| { let work = future::lazy(move |_| {
let api = client.runtime_api(); let api = client.runtime_api();
let global_validation = try_fr!(api.global_validation_schedule(&id));
let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) { let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) {
Some(local_validation) => local_validation, Some(local_validation) => local_validation,
None => return future::Either::Left(future::ok(())), None => return future::Either::Left(future::ok(())),
...@@ -297,6 +301,7 @@ fn build_collator_service<S, P, Extrinsic>( ...@@ -297,6 +301,7 @@ fn build_collator_service<S, P, Extrinsic>(
let collation_work = collate( let collation_work = collate(
relay_parent, relay_parent,
para_id, para_id,
global_validation,
local_validation, local_validation,
parachain_context, parachain_context,
key, key,
...@@ -427,6 +432,7 @@ mod tests { ...@@ -427,6 +432,7 @@ mod tests {
fn produce_candidate( fn produce_candidate(
&mut self, &mut self,
_relay_parent: Hash, _relay_parent: Hash,
_global: GlobalValidationSchedule,
_local_validation: LocalValidationData, _local_validation: LocalValidationData,
) -> Self::ProduceCandidate { ) -> Self::ProduceCandidate {
// send messages right back. // send messages right back.
......
...@@ -6,13 +6,18 @@ description = "Types and utilities for creating and working with parachains" ...@@ -6,13 +6,18 @@ description = "Types and utilities for creating and working with parachains"
edition = "2018" edition = "2018"
[dependencies] [dependencies]
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = [ "derive" ] } # note: special care is taken to avoid inclusion of `sp-io` externals when compiling
derive_more = { version = "0.99.2", optional = true } # this crate for WASM. This is critical to avoid forcing all parachain WASM into implementing
serde = { version = "1.0.102", default-features = false, features = [ "derive" ], optional = true } # various unnecessary Substrate-specific endpoints.
codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = [ "derive" ] }
sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
# all optional crates.
derive_more = { version = "0.99.2", optional = true }
serde = { version = "1.0.102", default-features = false, features = [ "derive" ], optional = true }
sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true, default-features = false }
sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
...@@ -22,14 +27,9 @@ log = { version = "0.4.8", optional = true } ...@@ -22,14 +27,9 @@ log = { version = "0.4.8", optional = true }
[target.'cfg(not(target_os = "unknown"))'.dependencies] [target.'cfg(not(target_os = "unknown"))'.dependencies]
shared_memory = { version = "0.10.0", optional = true } shared_memory = { version = "0.10.0", optional = true }
[dev-dependencies]
tiny-keccak = "1.5.0"
adder = { path = "../test-parachains/adder" }
halt = { path = "../test-parachains/halt" }
[features] [features]
default = ["std"] default = ["std"]
wasm-api = [] wasm-api = ["sp-runtime-interface"]
std = [ std = [
"codec/std", "codec/std",
"derive_more", "derive_more",
...@@ -39,6 +39,7 @@ std = [ ...@@ -39,6 +39,7 @@ std = [
"sp-core/std", "sp-core/std",
"parking_lot", "parking_lot",
"log", "log",
"sp-runtime-interface",
"sp-runtime-interface/std", "sp-runtime-interface/std",
"sp-externalities", "sp-externalities",
"sc-executor", "sc-executor",
......
...@@ -45,182 +45,9 @@ ...@@ -45,182 +45,9 @@
#[cfg(feature = "std")] #[cfg(feature = "std")]
pub mod wasm_executor; pub mod wasm_executor;
pub mod primitives;
mod wasm_api; mod wasm_api;
use sp_std::vec::Vec;
use codec::{Encode, Decode, CompactAs};
use sp_core::{RuntimeDebug, TypeId};
#[cfg(all(not(feature = "std"), feature = "wasm-api"))] #[cfg(all(not(feature = "std"), feature = "wasm-api"))]
pub use wasm_api::*; pub use wasm_api::*;
/// Validation parameters for evaluating the parachain validity function.
// TODO: balance downloads (https://github.com/paritytech/polkadot/issues/220)
#[derive(PartialEq, Eq, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Encode))]
pub struct ValidationParams {
/// The collation body.
pub block_data: Vec<u8>,
/// Previous head-data.
pub parent_head: Vec<u8>,
}
/// The result of parachain validation.
// TODO: egress and balance uploads (https://github.com/paritytech/polkadot/issues/220)
#[derive(PartialEq, Eq, Encode)]
#[cfg_attr(feature = "std", derive(Debug, Decode))]
pub struct ValidationResult {
/// New head data that should be included in the relay chain state.
pub head_data: Vec<u8>,
}
/// Unique identifier of a parachain.
#[derive(
Clone, CompactAs, Copy, Decode, Default, Encode, Eq,
Hash, Ord, PartialEq, PartialOrd, RuntimeDebug,
)]
#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, derive_more::Display))]
pub struct Id(u32);
impl TypeId for Id {
const TYPE_ID: [u8; 4] = *b"para";
}
/// Type for determining the active set of parachains.
pub trait ActiveThreads {
/// Return the current ordered set of `Id`s of active parathreads.
fn active_threads() -> Vec<Id>;
}
impl From<Id> for u32 {
fn from(x: Id) -> Self { x.0 }
}
impl From<u32> for Id {
fn from(x: u32) -> Self { Id(x) }
}
const USER_INDEX_START: u32 = 1000;
/// The ID of the first user (non-system) parachain.
pub const LOWEST_USER_ID: Id = Id(USER_INDEX_START);
impl Id {
/// Create an `Id`.
pub const fn new(id: u32) -> Self {
Self(id)
}
/// Returns `true` if this parachain runs with system-level privileges.
pub fn is_system(&self) -> bool { self.0 < USER_INDEX_START }
}
impl sp_std::ops::Add<u32> for Id {
type Output = Self;
fn add(self, other: u32) -> Self {
Self(self.0 + other)
}
}
// TODO: Remove all of this, move sp-runtime::AccountIdConversion to own crate and and use that.
// #360
struct TrailingZeroInput<'a>(&'a [u8]);
impl<'a> codec::Input for TrailingZeroInput<'a> {
fn remaining_len(&mut self) -> Result<Option<usize>, codec::Error> {
Ok(None)
}
fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> {
let len = into.len().min(self.0.len());
into[..len].copy_from_slice(&self.0[..len]);
for i in &mut into[len..] {
*i = 0;
}
self.0 = &self.0[len..];
Ok(())
}
}
/// This type can be converted into and possibly from an AccountId (which itself is generic).
pub trait AccountIdConversion<AccountId>: Sized {
/// Convert into an account ID. This is infallible.
fn into_account(&self) -> AccountId;
/// Try to convert an account ID into this type. Might not succeed.
fn try_from_account(a: &AccountId) -> Option<Self>;
}
/// Format is b"para" ++ encode(parachain ID) ++ 00.... where 00... is indefinite trailing
/// zeroes to fill AccountId.
impl<T: Encode + Decode + Default> AccountIdConversion<T> for Id {
fn into_account(&self) -> T {
(b"para", self).using_encoded(|b|
T::decode(&mut TrailingZeroInput(b))
).unwrap_or_default()
}
fn try_from_account(x: &T) -> Option<Self> {
x.using_encoded(|d| {
if &d[0..4] != b"para" { return None }
let mut cursor = &d[4..];
let result = Decode::decode(&mut cursor).ok()?;
if cursor.iter().all(|x| *x == 0) {
Some(result)
} else {
None
}
})
}
}
/// Which origin a parachain's message to the relay chain should be dispatched from.
#[derive(Clone, PartialEq, Eq, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug))]
#[repr(u8)]
pub enum ParachainDispatchOrigin {
/// As a simple `Origin::Signed`, using `ParaId::account_id` as its value. This is good when
/// interacting with standard modules such as `balances`.
Signed,
/// As the special `Origin::Parachain(ParaId)`. This is good when interacting with parachain-
/// aware modules which need to succinctly verify that the origin is a parachain.
Parachain,
/// As the simple, superuser `Origin::Root`. This can only be done on specially permissioned
/// parachains.
Root,
}
impl sp_std::convert::TryFrom<u8> for ParachainDispatchOrigin {
type Error = ();
fn try_from(x: u8) -> core::result::Result<ParachainDispatchOrigin, ()> {
const SIGNED: u8 = ParachainDispatchOrigin::Signed as u8;
const PARACHAIN: u8 = ParachainDispatchOrigin::Parachain as u8;
Ok(match x {
SIGNED => ParachainDispatchOrigin::Signed,
PARACHAIN => ParachainDispatchOrigin::Parachain,
_ => return Err(()),
})
}
}
/// A message from a parachain to its Relay Chain.
#[derive(Clone, PartialEq, Eq, Encode, Decode, sp_runtime_interface::pass_by::PassByCodec)]
#[cfg_attr(feature = "std", derive(Debug))]
pub struct UpwardMessage {
/// The origin for the message to be sent from.
pub origin: ParachainDispatchOrigin,
/// The message data.
pub data: Vec<u8>,
}
/// An incoming message.
#[derive(PartialEq, Eq, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Encode))]
pub struct IncomingMessage {
/// The source parachain.
pub source: Id,
/// The data of the message.
pub data: Vec<u8>,
}
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Primitive types which are strictly necessary from a parachain-execution point
//! of view.
use sp_std::vec::Vec;
use codec::{Encode, Decode, CompactAs};
use sp_core::{RuntimeDebug, TypeId};
#[cfg(feature = "std")]
use serde::{Serialize, Deserialize};
#[cfg(feature = "std")]
use sp_core::bytes;
/// The block number of the relay chain.
/// 32-bits will allow for 136 years of blocks assuming 1 block per second.
pub type RelayChainBlockNumber = u32;
/// Parachain head data included in the chain.
#[derive(PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Default))]
pub struct HeadData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
/// Parachain validation code.
#[derive(Default, PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
/// Parachain block data.
///
/// Contains everything required to validate para-block, may contain block and witness data.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
pub struct BlockData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
/// Unique identifier of a parachain.
#[derive(
Clone, CompactAs, Copy, Decode, Default, Encode, Eq,
Hash, Ord, PartialEq, PartialOrd, RuntimeDebug,
)]
#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, derive_more::Display))]
pub struct Id(u32);
impl TypeId for Id {
const TYPE_ID: [u8; 4] = *b"para";
}
impl From<Id> for u32 {
fn from(x: Id) -> Self { x.0 }
}
impl From<u32> for Id {
fn from(x: u32) -> Self { Id(x) }
}
const USER_INDEX_START: u32 = 1000;
/// The ID of the first user (non-system) parachain.
pub const LOWEST_USER_ID: Id = Id(USER_INDEX_START);
impl Id {
/// Create an `Id`.
pub const fn new(id: u32) -> Self {
Self(id)
}
/// Returns `true` if this parachain runs with system-level privileges.
pub fn is_system(&self) -> bool { self.0 < USER_INDEX_START }
}
impl sp_std::ops::Add<u32> for Id {
type Output = Self;