Unverified Commit f5535faf authored by Gavin Wood's avatar Gavin Wood Committed by GitHub
Browse files

Merged companions and update Subtrate (#882)



* expunge legacy code from polkadot-network

* mostly rip out old legacy protocol from service

* ensure validation work is spawned by incoming messages

* decouple availabliity store from network logic; clean up data flow

* av_store: test helpers and use futures-abort

* update polkadot-validation to pass n_validators when submitting chunks

* fallible erasure-chunk fetching

* implement `ErasureNetworking` for new network prot

* API for registering availability store in network

* fully integrate new network service into service

* fix validation tests

* scaffolding for porting collator over to new network

* track connected validators' peer IDs and distribute collators' collations

* helper in network for fetching all checked statements

* fix adder-collator

* actually register notifications protocol

* Update service/src/lib.rs

* Make needed changes to service

* Merge two companion PRs.

- #880
- #881

* Some effort towards compilation

* Fix

* remove `NetworkSpecialization` references from network

* fix compilation errors in service and collator

* ensure protocol name is valid

* Fixes

* Fix
Co-authored-by: asynchronous rob's avatarRobert Habermeier <rphmeier@gmail.com>
Co-authored-by: default avatarAshley <ashley.ruglys@gmail.com>
parent fd617eb6
Pipeline #81803 canceled with stages
in 18 minutes and 24 seconds
......@@ -8,3 +8,4 @@ runtime/wasm/target/
.vscode
polkadot.*
.DS_Store
.cargo
This diff is collapsed.
......@@ -66,7 +66,6 @@ use polkadot_cli::{
ProvideRuntimeApi, AbstractService, ParachainHost, IsKusama,
service::{self, Roles}
};
use polkadot_network::PolkadotProtocol;
pub use polkadot_cli::{VersionInfo, load_spec, service::Configuration};
pub use polkadot_validation::SignedStatement;
pub use polkadot_primitives::parachain::CollatorId;
......@@ -212,7 +211,7 @@ fn build_collator_service<S, P, Extrinsic>(
build_parachain_context: P,
) -> Result<S, polkadot_service::Error>
where
S: AbstractService<Block = service::Block, NetworkSpecialization = PolkadotProtocol>,
S: AbstractService<Block = service::Block>,
sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi>: ProvideRuntimeApi<Block>,
<sc_client::Client<S::Backend, S::CallExecutor, service::Block, S::RuntimeApi> as ProvideRuntimeApi<Block>>::Api:
RuntimeApiCollection<
......
......@@ -52,7 +52,7 @@
use sp_runtime::traits::{BlakeTwo256, Hash as HashT};
use sp_blockchain::Error as ClientError;
use sc_network::{config::Roles, PeerId, ReputationChange};
use sc_network::{NetworkService as SubstrateNetworkService, specialization::NetworkSpecialization};
use sc_network::NetworkService;
use sc_network_gossip::{
ValidationResult as GossipValidationResult,
ValidatorContext, MessageIntent,
......@@ -266,11 +266,11 @@ pub(crate) fn attestation_topic(parent_hash: Hash) -> Hash {
// NOTE: since RegisteredMessageValidator is meant to be a type-safe proof
// that we've actually done the registration, this should be the only way
// to construct it outside of tests.
pub fn register_validator<C: ChainContext + 'static, S: NetworkSpecialization<Block>>(
service: Arc<SubstrateNetworkService<Block, S, Hash>>,
pub fn register_validator<C: ChainContext + 'static>(
service: Arc<NetworkService<Block, Hash>>,
chain: C,
executor: &impl futures::task::Spawn,
) -> RegisteredMessageValidator<S>
) -> RegisteredMessageValidator
{
let s = service.clone();
let report_handle = Box::new(move |peer: &PeerId, cost_benefit: ReputationChange| {
......@@ -344,25 +344,16 @@ impl NewLeafActions {
/// A registered message validator.
///
/// Create this using `register_validator`.
pub struct RegisteredMessageValidator<S: NetworkSpecialization<Block>> {
#[derive(Clone)]
pub struct RegisteredMessageValidator {
inner: Arc<MessageValidator<dyn ChainContext>>,
// Note: this is always `Some` in real code and `None` in tests.
service: Option<Arc<SubstrateNetworkService<Block, S, Hash>>>,
service: Option<Arc<NetworkService<Block, Hash>>>,
// Note: this is always `Some` in real code and `None` in tests.
gossip_engine: Option<sc_network_gossip::GossipEngine<Block>>,
}
impl<S: NetworkSpecialization<Block>> Clone for RegisteredMessageValidator<S> {
fn clone(&self) -> Self {
RegisteredMessageValidator {
inner: self.inner.clone(),
service: self.service.clone(),
gossip_engine: self.gossip_engine.clone(),
}
}
}
impl<S: NetworkSpecialization<Block>> RegisteredMessageValidator<S> {
impl RegisteredMessageValidator {
/// Register an availabilty store the gossip service can query.
pub(crate) fn register_availability_store(&self, availability_store: av_store::Store) {
self.inner.inner.write().availability_store = Some(availability_store);
......@@ -437,7 +428,7 @@ impl<S: NetworkSpecialization<Block>> RegisteredMessageValidator<S> {
}
}
impl<S: NetworkSpecialization<Block>> GossipService for RegisteredMessageValidator<S> {
impl GossipService for RegisteredMessageValidator {
fn gossip_messages_for(&self, topic: Hash) -> GossipMessageStream {
RegisteredMessageValidator::gossip_messages_for(self, topic)
}
......
......@@ -24,14 +24,8 @@ use polkadot_primitives::{Block, Hash, BlakeTwo256, HashT};
pub mod legacy;
pub mod protocol;
sc_network::construct_simple_protocol! {
/// Stub until https://github.com/paritytech/substrate/pull/4665 is merged
#[derive(Clone)]
pub struct PolkadotProtocol where Block = Block { }
}
/// Specialization of the network service for the polkadot protocol.
pub type PolkadotNetworkService = sc_network::NetworkService<Block, PolkadotProtocol, Hash>;
/// Specialization of the network service for the polkadot block type.
pub type PolkadotNetworkService = sc_network::NetworkService<Block, Hash>;
mod cost {
use sc_network::ReputationChange as Rep;
......
......@@ -50,7 +50,7 @@ use std::time::Duration;
use super::{cost, benefit, PolkadotNetworkService};
use crate::legacy::collator_pool::Role as CollatorRole;
use crate::legacy::gossip::{GossipMessage, ErasureChunkMessage};
use crate::legacy::gossip::{GossipMessage, ErasureChunkMessage, RegisteredMessageValidator};
/// The current protocol version.
pub const VERSION: u32 = 1;
......@@ -60,7 +60,7 @@ pub const MIN_SUPPORTED_VERSION: u32 = 1;
/// The engine ID of the polkadot network protocol.
pub const POLKADOT_ENGINE_ID: sp_runtime::ConsensusEngineId = *b"dot2";
/// The protocol name.
pub const POLKADOT_PROTOCOL_NAME: &[u8] = b"dot2-proto";
pub const POLKADOT_PROTOCOL_NAME: &[u8] = b"/polkadot/1";
pub use crate::legacy::gossip::ChainContext;
......@@ -305,8 +305,6 @@ struct ConsensusNetworkingInstance {
_drop_signal: exit_future::Signal,
}
type RegisteredMessageValidator = crate::legacy::gossip::RegisteredMessageValidator<crate::PolkadotProtocol>;
/// Protocol configuration.
#[derive(Default)]
pub struct Config {
......
......@@ -12,6 +12,7 @@ serde = { version = "1.0.102", default-features = false, features = [ "derive" ]
rstd = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "polkadot-master", default-features = false }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", default-features = false }
sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", default-features = false }
sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", default-features = false }
sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", optional = true }
sc-executor = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", optional = true }
sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master", optional = true }
......
......@@ -24,6 +24,8 @@ use std::any::{TypeId, Any};
use crate::{ValidationParams, ValidationResult, UpwardMessage};
use codec::{Decode, Encode};
use sp_core::storage::{ChildStorageKey, ChildInfo};
use sp_core::traits::CallInWasm;
use sp_wasm_interface::HostFunctions as _;
#[cfg(not(target_os = "unknown"))]
pub use validation_host::{run_worker, EXECUTION_TIMEOUT_SEC};
......@@ -156,15 +158,18 @@ pub fn validate_candidate_internal<E: Externalities + 'static>(
) -> Result<ValidationResult, Error> {
let mut ext = ValidationExternalities(ParachainExt::new(externalities));
let res = sc_executor::call_in_wasm::<HostFunctions>(
"validate_block",
encoded_call_data,
let executor = sc_executor::WasmExecutor::new(
sc_executor::WasmExecutionMethod::Interpreted,
&mut ext,
validation_code,
// TODO: Make sure we don't use more than 1GB: https://github.com/paritytech/polkadot/issues/699
1024,
Some(1024),
HostFunctions::host_functions(),
false,
);
let res = executor.call_in_wasm(
validation_code,
"validate_block",
encoded_call_data,
&mut ext,
)?;
ValidationResult::decode(&mut &res[..]).map_err(|_| Error::BadReturn.into())
......
......@@ -330,6 +330,7 @@ mod tests {
parameter_types! {
pub const ExistentialDeposit: u64 = 1;
pub const CreationFee: u64 = 0;
pub const MinVestedTransfer: u64 = 0;
}
impl balances::Trait for Test {
......@@ -344,6 +345,7 @@ mod tests {
type Event = ();
type Currency = Balances;
type BlockNumberToBalance = Identity;
type MinVestedTransfer = MinVestedTransfer;
}
parameter_types!{
......
......@@ -336,11 +336,16 @@ impl democracy::Trait for Runtime {
type Slash = Treasury;
}
parameter_types! {
pub const CouncilMotionDuration: BlockNumber = 3 * DAYS;
}
type CouncilCollective = collective::Instance1;
impl collective::Trait<CouncilCollective> for Runtime {
type Origin = Origin;
type Proposal = Call;
type Event = Event;
type MotionDuration = CouncilMotionDuration;
}
parameter_types! {
......@@ -367,11 +372,16 @@ impl elections_phragmen::Trait for Runtime {
type TermDuration = TermDuration;
}
parameter_types! {
pub const TechnicalMotionDuration: BlockNumber = 3 * DAYS;
}
type TechnicalCollective = collective::Instance2;
impl collective::Trait<TechnicalCollective> for Runtime {
type Origin = Origin;
type Proposal = Call;
type Event = Event;
type MotionDuration = TechnicalMotionDuration;
}
impl membership::Trait<membership::Instance1> for Runtime {
......@@ -380,6 +390,7 @@ impl membership::Trait<membership::Instance1> for Runtime {
type RemoveOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type SwapOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type ResetOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type PrimeOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type MembershipInitialized = TechnicalCommittee;
type MembershipChanged = TechnicalCommittee;
}
......@@ -599,10 +610,15 @@ impl society::Trait for Runtime {
type ChallengePeriod = ChallengePeriod;
}
parameter_types! {
pub const MinVestedTransfer: Balance = 100 * DOLLARS;
}
impl vesting::Trait for Runtime {
type Event = Event;
type Currency = Balances;
type BlockNumberToBalance = ConvertInto;
type MinVestedTransfer = MinVestedTransfer;
}
construct_runtime! {
......
......@@ -343,11 +343,16 @@ impl democracy::Trait for Runtime {
type Slash = Treasury;
}
parameter_types! {
pub const CouncilMotionDuration: BlockNumber = 7 * DAYS;
}
type CouncilCollective = collective::Instance1;
impl collective::Trait<CouncilCollective> for Runtime {
type Origin = Origin;
type Proposal = Call;
type Event = Event;
type MotionDuration = CouncilMotionDuration;
}
parameter_types! {
......@@ -375,11 +380,16 @@ impl elections_phragmen::Trait for Runtime {
type TermDuration = TermDuration;
}
parameter_types! {
pub const TechnicalMotionDuration: BlockNumber = 7 * DAYS;
}
type TechnicalCollective = collective::Instance2;
impl collective::Trait<TechnicalCollective> for Runtime {
type Origin = Origin;
type Proposal = Call;
type Event = Event;
type MotionDuration = TechnicalMotionDuration;
}
impl membership::Trait<membership::Instance1> for Runtime {
......@@ -388,6 +398,7 @@ impl membership::Trait<membership::Instance1> for Runtime {
type RemoveOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type SwapOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type ResetOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type PrimeOrigin = collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
type MembershipInitialized = TechnicalCommittee;
type MembershipChanged = TechnicalCommittee;
}
......@@ -525,10 +536,15 @@ impl claims::Trait for Runtime {
type Prefix = Prefix;
}
parameter_types! {
pub const MinVestedTransfer: Balance = 100 * DOLLARS;
}
impl vesting::Trait for Runtime {
type Event = Event;
type Currency = Balances;
type BlockNumberToBalance = ConvertInto;
type MinVestedTransfer = MinVestedTransfer;
}
impl sudo::Trait for Runtime {
......
......@@ -23,7 +23,7 @@ use std::sync::Arc;
use std::time::Duration;
use polkadot_primitives::{parachain, Hash, BlockId, AccountId, Nonce, Balance};
use polkadot_network::legacy::gossip::Known;
use polkadot_network::{protocol as network_protocol, PolkadotProtocol as StubSpecialization};
use polkadot_network::protocol as network_protocol;
use service::{error::{Error as ServiceError}, ServiceBuilder};
use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
use inherents::InherentDataProviders;
......@@ -33,7 +33,7 @@ pub use service::{
AbstractService, Roles, PruningMode, TransactionPoolOptions, Error, RuntimeGenesis, ServiceBuilderCommand,
TFullClient, TLightClient, TFullBackend, TLightBackend, TFullCallExecutor, TLightCallExecutor,
};
pub use service::config::{DatabaseConfig, full_version_from_strs};
pub use service::config::{DatabaseConfig, PrometheusConfig, full_version_from_strs};
pub use sc_executor::NativeExecutionDispatch;
pub use sc_client::{ExecutionStrategy, CallExecutor, Client};
pub use sc_client_api::backend::Backend;
......@@ -43,7 +43,6 @@ pub use consensus_common::SelectChain;
pub use polkadot_primitives::parachain::{CollatorId, ParachainHost};
pub use polkadot_primitives::Block;
pub use sp_runtime::traits::{Block as BlockT, self as runtime_traits, BlakeTwo256};
pub use sc_network::specialization::NetworkSpecialization;
pub use chain_spec::ChainSpec;
#[cfg(not(target_os = "unknown"))]
pub use consensus::run_validation_worker;
......@@ -126,12 +125,23 @@ impl IsKusama for ChainSpec {
}
}
// If we're using prometheus, use a registry with a prefix of `polkadot`.
fn set_prometheus_registry(config: &mut Configuration) -> Result<(), ServiceError> {
if let Some(PrometheusConfig { registry, .. }) = config.prometheus_config.as_mut() {
*registry = Registry::new_custom(Some("polkadot".into()), None)?;
}
Ok(())
}
/// Starts a `ServiceBuilder` for a full service.
///
/// Use this macro if you don't actually need the full service, but just the builder in order to
/// be able to perform chain operations.
macro_rules! new_full_start {
($config:expr, $runtime:ty, $executor:ty) => {{
set_prometheus_registry(&mut $config)?;
let mut import_setup = None;
let inherent_data_providers = inherents::InherentDataProviders::new();
let builder = service::ServiceBuilder::new_full::<
......@@ -150,7 +160,7 @@ macro_rules! new_full_start {
.ok_or_else(|| service::Error::SelectChainRequired)?;
let (grandpa_block_import, grandpa_link) =
grandpa::block_import(
client.clone(), &*client, select_chain
client.clone(), &(client.clone() as Arc<_>), select_chain
)?;
let justification_import = grandpa_block_import.clone();
......@@ -174,10 +184,7 @@ macro_rules! new_full_start {
})?
.with_rpc_extensions(|builder| -> Result<polkadot_rpc::RpcExtension, _> {
Ok(polkadot_rpc::create_full(builder.client().clone(), builder.pool()))
})?
.with_prometheus_registry(
Registry::new_custom(Some("polkadot".into()), None)?
);
})?;
(builder, import_setup, inherent_data_providers)
}}
......@@ -210,7 +217,6 @@ pub fn polkadot_new_full(
impl AbstractService<
Block = Block,
RuntimeApi = polkadot_runtime::RuntimeApi,
NetworkSpecialization = StubSpecialization,
Backend = TFullBackend<Block>,
SelectChain = LongestChain<TFullBackend<Block>, Block>,
CallExecutor = TFullCallExecutor<Block, PolkadotExecutor>,
......@@ -233,7 +239,6 @@ pub fn kusama_new_full(
impl AbstractService<
Block = Block,
RuntimeApi = kusama_runtime::RuntimeApi,
NetworkSpecialization = StubSpecialization,
Backend = TFullBackend<Block>,
SelectChain = LongestChain<TFullBackend<Block>, Block>,
CallExecutor = TFullCallExecutor<Block, KusamaExecutor>,
......@@ -253,7 +258,7 @@ pub struct FullNodeHandles {
/// Builds a new service for a full client.
pub fn new_full<Runtime, Dispatch, Extrinsic>(
config: Configuration,
mut config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
......@@ -263,7 +268,6 @@ pub fn new_full<Runtime, Dispatch, Extrinsic>(
impl AbstractService<
Block = Block,
RuntimeApi = Runtime,
NetworkSpecialization = StubSpecialization,
Backend = TFullBackend<Block>,
SelectChain = LongestChain<TFullBackend<Block>, Block>,
CallExecutor = TFullCallExecutor<Block, Dispatch>,
......@@ -280,6 +284,7 @@ pub fn new_full<Runtime, Dispatch, Extrinsic>(
<Runtime::RuntimeApi as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
{
use sc_network::Event;
use sc_client_api::ExecutorProvider;
use futures::stream::StreamExt;
let is_collator = collating_for.is_some();
......@@ -307,10 +312,10 @@ pub fn new_full<Runtime, Dispatch, Extrinsic>(
let backend = builder.backend().clone();
let service = builder
.with_network_protocol(|_config| Ok(StubSpecialization::new()))?
.with_finality_proof_provider(|client, backend|
Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
)?
.with_finality_proof_provider(|client, backend| {
let provider = client as Arc<dyn grandpa::StorageAndProofProvider<_, _>>;
Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _)
})?
.build()?;
let (block_import, link_half, babe_link) = import_setup.take()
......@@ -478,7 +483,6 @@ pub fn new_full<Runtime, Dispatch, Extrinsic>(
link: link_half,
network: service.network(),
inherent_data_providers: inherent_data_providers.clone(),
on_exit: service.on_exit(),
telemetry_on_connect: Some(service.telemetry_on_connect_stream()),
voting_rule: grandpa::VotingRulesBuilder::default().build(),
prometheus_registry: service.prometheus_registry(),
......@@ -507,7 +511,6 @@ pub fn polkadot_new_light(
-> Result<impl AbstractService<
Block = Block,
RuntimeApi = polkadot_runtime::RuntimeApi,
NetworkSpecialization = StubSpecialization,
Backend = TLightBackend<Block>,
SelectChain = LongestChain<TLightBackend<Block>, Block>,
CallExecutor = TLightCallExecutor<Block, PolkadotExecutor>,
......@@ -523,7 +526,6 @@ pub fn kusama_new_light(
-> Result<impl AbstractService<
Block = Block,
RuntimeApi = kusama_runtime::RuntimeApi,
NetworkSpecialization = StubSpecialization,
Backend = TLightBackend<Block>,
SelectChain = LongestChain<TLightBackend<Block>, Block>,
CallExecutor = TLightCallExecutor<Block, KusamaExecutor>,
......@@ -552,12 +554,11 @@ type TLocalLightClient<Runtime, Dispatch> = Client<
/// Builds a new service for a light client.
pub fn new_light<Runtime, Dispatch, Extrinsic>(
config: Configuration,
mut config: Configuration,
)
-> Result<impl AbstractService<
Block = Block,
RuntimeApi = Runtime,
NetworkSpecialization = StubSpecialization,
Backend = TLightBackend<Block>,
SelectChain = LongestChain<TLightBackend<Block>, Block>,
CallExecutor = TLightCallExecutor<Block, Dispatch>,
......@@ -575,6 +576,8 @@ where
TLocalLightClient<Runtime, Dispatch>,
>,
{
set_prometheus_registry(&mut config)?;
let inherent_data_providers = InherentDataProviders::new();
ServiceBuilder::new_light::<Block, Runtime, Dispatch>(config)?
......@@ -595,7 +598,7 @@ where
.map(|fetcher| fetcher.checker().clone())
.ok_or_else(|| "Trying to start light import queue without active fetch checker")?;
let grandpa_block_import = grandpa::light_block_import(
client.clone(), backend, &*client, Arc::new(fetch_checker)
client.clone(), backend, &(client.clone() as Arc<_>), Arc::new(fetch_checker)
)?;
let finality_proof_import = grandpa_block_import.clone();
......@@ -620,10 +623,10 @@ where
Ok((import_queue, finality_proof_request_builder))
})?
.with_network_protocol(|_config| Ok(StubSpecialization::new()))?
.with_finality_proof_provider(|client, backend|
Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
)?
.with_finality_proof_provider(|client, backend| {
let provider = client as Arc<dyn grandpa::StorageAndProofProvider<_, _>>;
Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _)
})?
.with_rpc_extensions(|builder|
-> Result<polkadot_rpc::RpcExtension, _> {
let fetcher = builder.fetcher()
......@@ -633,8 +636,5 @@ where
Ok(polkadot_rpc::create_light(builder.client().clone(), remote_blockchain, fetcher, builder.pool()))
})?
.with_prometheus_registry(
Registry::new_custom(Some("polkadot".into()), None)?
)
.build()
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment