Skip to content
GitLab
Commit 7f573736 authored by Cecile Tonglet's avatar Cecile Tonglet
Browse files

Merge commit 00202d37 (conflicts) 

Parent branch: origin/master
Forked at: bf7ccb84
parents 1fd29876 00202d37
Expand all Hide whitespace changes
Inline Side-by-side
Cargo.lock
View file @ 7f573736
......@@ -2683,7 +2683,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
......@@ -4729,7 +4729,6 @@ dependencies = [
"nix 0.17.0",
"parity-util-mem",
"polkadot-cli",
"polkadot-collator",
"polkadot-service",
"tempfile",
]
......@@ -4785,32 +4784,6 @@ dependencies = [
"streamunordered",
]
[[package]]
name = "polkadot-availability-store"
version = "0.8.22"
dependencies = [
"derive_more 0.99.9",
"exit-future",
"futures 0.3.5",
"kvdb",
"kvdb-memorydb",
"kvdb-rocksdb",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-erasure-coding",
"polkadot-primitives",
"sc-client-api",
"sc-keystore",
"sc-network",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
"tokio 0.2.21",
]
[[package]]
name = "polkadot-cli"
version = "0.8.22"
......@@ -4837,34 +4810,6 @@ dependencies = [
"wasm-bindgen-futures",
]
[[package]]
name = "polkadot-collator"
version = "0.8.22"
dependencies = [
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"polkadot-cli",
"polkadot-network",
"polkadot-primitives",
"polkadot-service",
"polkadot-service-new",
"polkadot-validation",
"sc-cli",
"sc-client-api",
"sc-executor",
"sc-network",
"sc-service",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-keyring",
"sp-runtime",
"tokio 0.2.21",
]
[[package]]
name = "polkadot-core-primitives"
version = "0.7.30"
......@@ -4887,36 +4832,6 @@ dependencies = [
"sp-trie",
]
[[package]]
name = "polkadot-network"
version = "0.8.22"
dependencies = [
"arrayvec 0.4.12",
"bytes 0.5.5",
"derive_more 0.14.1",
"exit-future",
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-erasure-coding",
"polkadot-primitives",
"polkadot-validation",
"rand 0.7.3",
"sc-network",
"sc-network-gossip",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-keyring",
"sp-runtime",
"sp-state-machine",
"wasm-timer",
]
[[package]]
name = "polkadot-network-bridge"
version = "0.1.0"
......@@ -4938,27 +4853,6 @@ dependencies = [
"streamunordered",
]
[[package]]
name = "polkadot-network-test"
version = "0.8.22"
dependencies = [
"futures 0.3.5",
"log 0.4.11",
"parking_lot 0.10.2",
"polkadot-test-runtime-client",
"rand 0.7.3",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-network",
"sc-network-test",
"sc-service",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
]
[[package]]
name = "polkadot-node-collation-generation"
version = "0.1.0"
......@@ -5396,7 +5290,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
"trie-db",
]
......@@ -5508,8 +5402,6 @@ dependencies = [
"pallet-transaction-payment-rpc-runtime-api",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-network",
"polkadot-primitives",
"polkadot-rpc",
"polkadot-runtime",
......@@ -5568,7 +5460,6 @@ dependencies = [
"pallet-transaction-payment-rpc-runtime-api",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-network",
"polkadot-node-core-proposer",
"polkadot-node-subsystem",
"polkadot-overseer",
......@@ -5699,7 +5590,7 @@ dependencies = [
"sp-transaction-pool",
"sp-trie",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
......@@ -5739,14 +5630,11 @@ dependencies = [
"pallet-balances",
"pallet-staking",
"pallet-transaction-payment",
"polkadot-availability-store",
"polkadot-network",
"polkadot-primitives",
"polkadot-rpc",
"polkadot-runtime-common",
"polkadot-service",
"polkadot-test-runtime",
"polkadot-validation",
"rand 0.7.3",
"sc-authority-discovery",
"sc-chain-spec",
......@@ -5780,25 +5668,16 @@ dependencies = [
name = "polkadot-validation"
version = "0.8.22"
dependencies = [
"ansi_term 0.12.1",
"bitvec",
"derive_more 0.14.1",
"exit-future",
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-erasure-coding",
"polkadot-parachain",
"polkadot-primitives",
"polkadot-statement-table",
"sc-basic-authorship",
"sc-block-builder",
"sc-client-api",
"sc-finality-grandpa",
"sc-keystore",
"sp-api",
"sp-blockchain",
"sp-consensus",
......@@ -5811,7 +5690,6 @@ dependencies = [
"sp-transaction-pool",
"sp-trie",
"substrate-prometheus-endpoint",
"tokio 0.2.21",
]
[[package]]
......@@ -6515,7 +6393,7 @@ dependencies = [
"sp-std",
"sp-transaction-pool",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
]
[[package]]
......@@ -7238,33 +7116,6 @@ dependencies = [
"wasm-timer",
]
[[package]]
name = "sc-network-test"
version = "0.8.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"env_logger",
"futures 0.3.5",
"futures-timer 3.0.2",
"libp2p",
"log 0.4.11",
"parking_lot 0.10.2",
"rand 0.7.3",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-network",
"sc-service",
"sp-blockchain",
"sp-consensus",
"sp-consensus-babe",
"sp-core",
"sp-runtime",
"substrate-test-runtime",
"substrate-test-runtime-client",
"tempfile",
]
[[package]]
name = "sc-offchain"
version = "2.0.0-rc6"
......@@ -8135,20 +7986,6 @@ dependencies = [
"wasm-timer",
]
[[package]]
name = "sp-consensus-aura"
version = "0.8.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"parity-scale-codec",
"sp-api",
"sp-application-crypto",
"sp-inherents",
"sp-runtime",
"sp-std",
"sp-timestamp",
]
[[package]]
name = "sp-consensus-babe"
version = "0.8.0-rc6"
......@@ -8819,67 +8656,6 @@ dependencies = [
"sp-state-machine",
]
[[package]]
name = "substrate-test-runtime"
version = "2.0.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"cfg-if",
"frame-executive",
"frame-support",
"frame-system",
"frame-system-rpc-runtime-api",
"log 0.4.11",
"memory-db",
"pallet-babe",
"pallet-timestamp",
"parity-scale-codec",
"parity-util-mem",
"sc-service",
"serde",
"sp-api",
"sp-application-crypto",
"sp-block-builder",
"sp-consensus-aura",
"sp-consensus-babe",
"sp-core",
"sp-finality-grandpa",
"sp-inherents",
"sp-io",
"sp-keyring",
"sp-offchain",
"sp-runtime",
"sp-runtime-interface",
"sp-session",
"sp-std",
"sp-transaction-pool",
"sp-trie",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (git+https://github.com/paritytech/substrate)",
"trie-db",
]
[[package]]
name = "substrate-test-runtime-client"
version = "2.0.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"futures 0.3.5",
"parity-scale-codec",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-light",
"sc-service",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
"substrate-test-client",
"substrate-test-runtime",
]
[[package]]
name = "substrate-test-utils"
version = "2.0.0-rc6"
......@@ -8900,11 +8676,6 @@ dependencies = [
"syn 1.0.33",
]
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.6"
......@@ -9012,33 +8783,15 @@ dependencies = [
"polkadot-parachain",
"sp-io",
"sp-std",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
[[package]]
name = "test-parachain-adder-collator"
version = "0.1.0"
dependencies = [
"futures 0.3.5",
"parity-scale-codec",
"parking_lot 0.10.2",
"polkadot-collator",
"polkadot-parachain",
"polkadot-primitives",
"polkadot-service",
"sc-client-api",
"sp-api",
"sp-core",
"sp-runtime",
"test-parachain-adder",
]
[[package]]
name = "test-parachain-halt"
version = "0.8.22"
dependencies = [
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
]
[[package]]
......@@ -10209,7 +9962,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
......
Cargo.toml
View file @ 7f573736
......@@ -10,8 +10,6 @@ edition = "2018"
[dependencies]
cli = { package = "polkadot-cli", path = "cli" }
# It looks like this is the only way to pass features to it
collator = { package = "polkadot-collator", path = "collator" }
futures = "0.3.4"
service = { package = "polkadot-service", path = "service" }
parity-util-mem = { version = "*", default-features = false, features = ["jemalloc-global"] }
......@@ -23,13 +21,9 @@ tempfile = "3.1.0"
[workspace]
members = [
"availability-store",
"cli",
"collator",
"core-primitives",
"erasure-coding",
"network",
"network/test",
"primitives",
"runtime/common",
"runtime/parachains",
......@@ -69,7 +63,6 @@ members = [
"parachain/test-parachains",
"parachain/test-parachains/adder",
"parachain/test-parachains/adder/collator",
]
[badges]
......@@ -83,5 +76,4 @@ panic = "unwind"
runtime-benchmarks=["cli/runtime-benchmarks"]
service-rewr= [
"cli/service-rewr",
"collator/service-rewr",
]
availability-store/Cargo.toml deleted 100644 → 0
View file @ 1fd29876
[package]
name = "polkadot-availability-store"
description = "Persistent database for parachain data"
version = "0.8.22"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
polkadot-primitives = { path = "../primitives" }
polkadot-erasure-coding = { path = "../erasure-coding" }
parking_lot = "0.9.0"
derive_more = "0.99"
log = "0.4.8"
futures = "0.3.4"
tokio = { version = "0.2.13", features = ["rt-core"] }
exit-future = "0.2.0"
codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive"] }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master", version = "2.0.0-rc5" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
keystore = { package = "sc-keystore", git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
kvdb = "0.7.0"
kvdb-memorydb = "0.7.0"
[target.'cfg(not(target_os = "unknown"))'.dependencies]
kvdb-rocksdb = "0.9.0"
availability-store/src/lib.rs deleted 100644 → 0
View file @ 1fd29876
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Persistent database for parachain data: PoV block data, erasure-coding chunks and outgoing messages.
//!
//! This will be written into during the block validation pipeline, and queried
//! by networking code in order to circulate required data and maintain availability
//! of it.
#![warn(missing_docs)]
use futures::prelude::*;
use futures::channel::{mpsc, oneshot};
use keystore::KeyStorePtr;
use polkadot_primitives::v0::{
Hash, Block,
PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
ParachainHost, AvailableData, OmittedValidationData,
};
use sp_runtime::traits::HashFor;
use sp_blockchain::Result as ClientResult;
use client::{
BlockchainEvents, BlockBackend,
};
use sp_api::{ApiExt, ProvideRuntimeApi};
use codec::{Encode, Decode};
use sp_core::traits::SpawnNamed;
use log::warn;
use std::sync::Arc;
use std::collections::HashSet;
use std::path::PathBuf;
use std::io;
use std::pin::Pin;
mod worker;
mod store;
pub use worker::AvailabilityBlockImport;
pub use store::AwaitedFrontierEntry;
use worker::{
Worker, WorkerHandle, IncludedParachainBlocks, WorkerMsg, MakeAvailable, Chunks
};
use store::Store as InnerStore;
const LOG_TARGET: &str = "availability";
/// Configuration for the availability store.
pub struct Config {
/// Cache size in bytes. If `None` default is used.
pub cache_size: Option<usize>,
/// Path to the database.
pub path: PathBuf,
}
/// An abstraction around networking for the availablity-store.
///
/// Currently it is not possible to use the networking code in the availability store
/// core directly due to a number of loop dependencies it requires:
///
/// `availability-store` -> `network` -> `availability-store`
///
/// `availability-store` -> `network` -> `validation` -> `availability-store`
///
/// So we provide this trait that gets implemented for a type in
/// the [`network`] module or a mock in tests.
///
/// [`network`]: ../polkadot_network/index.html
pub trait ErasureNetworking {
/// Errors that can occur when fetching erasure chunks.
type Error: std::fmt::Debug + 'static;
/// Fetch an erasure chunk from the networking service.
fn fetch_erasure_chunk(
&self,
candidate_hash: &Hash,
index: u32,
) -> Pin<Box<dyn Future<Output = Result<ErasureChunk, Self::Error>> + Send>>;
/// Distributes an erasure chunk to the correct validator node.
fn distribute_erasure_chunk(
&self,
candidate_hash: Hash,
chunk: ErasureChunk,
);
}
/// Data that, when combined with an `AbridgedCandidateReceipt`, is enough
/// to fully re-execute a block.
#[derive(Debug, Encode, Decode, PartialEq)]
pub struct ExecutionData {
/// The `PoVBlock`.
pub pov_block: PoVBlock,
/// The data omitted from the `AbridgedCandidateReceipt`.
pub omitted_validation: OmittedValidationData,
}
/// Handle to the availability store.
///
/// This provides a proxying API that
/// * in case of write operations provides async methods that send data to
/// the background worker and resolve when that data is processed by the worker
/// * in case of read opeartions queries the underlying storage synchronously.
#[derive(Clone)]
pub struct Store {
inner: InnerStore,
worker: Arc<WorkerHandle>,
to_worker: mpsc::UnboundedSender<WorkerMsg>,
}
impl Store {
/// Create a new `Store` with given config on disk.
///
/// Creating a store among other things starts a background worker thread that
/// handles most of the write operations to the storage.
#[cfg(not(target_os = "unknown"))]
pub fn new<EN>(config: Config, network: EN) -> io::Result<Self>
where EN: ErasureNetworking + Send + Sync + Clone + 'static
{
let inner = InnerStore::new(config)?;
let worker = Arc::new(Worker::start(inner.clone(), network));
let to_worker = worker.to_worker().clone();
Ok(Self {
inner,
worker,
to_worker,
})
}
/// Create a new in-memory `Store`. Useful for tests.
///
/// Creating a store among other things starts a background worker thread
/// that handles most of the write operations to the storage.
pub fn new_in_memory<EN>(network: EN) -> Self
where EN: ErasureNetworking + Send + Sync + Clone + 'static
{
let inner = InnerStore::new_in_memory();
let worker = Arc::new(Worker::start(inner.clone(), network));
let to_worker = worker.to_worker().clone();
Self {
inner,
worker,
to_worker,
}
}
/// Obtain a [`BlockImport`] implementation to import blocks into this store.
///
/// This block import will act upon all newly imported blocks sending information
/// about parachain heads included in them to this `Store`'s background worker.
/// The user may create multiple instances of [`BlockImport`]s with this call.
///
/// [`BlockImport`]: https://substrate.dev/rustdocs/v1.0/substrate_consensus_common/trait.BlockImport.html
pub fn block_import<I, P>(
&self,