Unverified Commit e75ae5b7 authored by Bastian Köcher's avatar Bastian Köcher Committed by GitHub
Browse files

Prepare Polkadot to be used by Cumulus (#1697)

* Prepare Polkadot to be used by Cumulus

This begins to make Polkadot usable from Cumulus.

* Remove old test

* Reset some changes

* More reverts

* MOARE

* Don't use `unimplemented!`
parent 61012c7c
Pipeline #107191 passed with stages
in 17 minutes and 49 seconds
......@@ -5099,6 +5099,7 @@ dependencies = [
"sp-core",
"sp-externalities",
"sp-io",
"sp-runtime",
"sp-std",
"sp-wasm-interface",
]
......@@ -5449,6 +5450,7 @@ dependencies = [
"sp-offchain",
"sp-runtime",
"sp-session",
"sp-storage",
"sp-transaction-pool",
"sp-trie",
"substrate-prometheus-endpoint",
......
......@@ -31,6 +31,7 @@ sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "mas
sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
grandpa = { package = "sc-finality-grandpa", git = "https://github.com/paritytech/substrate", branch = "master" }
......
......@@ -16,38 +16,335 @@
//! Polkadot Client meta trait
use sp_api::{ProvideRuntimeApi, ConstructRuntimeApi, CallApiAt};
use std::sync::Arc;
use sp_api::{ProvideRuntimeApi, CallApiAt, NumberFor};
use sp_blockchain::HeaderBackend;
use sp_runtime::traits::Block as BlockT;
use sc_client_api::{Backend as BackendT, BlockchainEvents};
use sp_runtime::{
Justification, generic::{BlockId, SignedBlock}, traits::{Block as BlockT, BlakeTwo256},
};
use sc_client_api::{Backend as BackendT, BlockchainEvents, KeyIterator};
use sp_storage::{StorageData, StorageKey, ChildInfo, PrefixedStorageKey};
use polkadot_primitives::v1::{Block, ParachainHost, AccountId, Nonce, Balance};
use consensus_common::BlockStatus;
/// Polkadot client abstraction, this super trait only pulls in functionality required for
/// polkadot internal crates like polkadot-collator.
pub trait PolkadotClient<Block, Backend, Runtime>:
/// A set of APIs that polkadot-like runtimes must implement.
pub trait RuntimeApiCollection:
sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
+ sp_api::ApiExt<Block, Error = sp_blockchain::Error>
+ babe_primitives::BabeApi<Block>
+ grandpa_primitives::GrandpaApi<Block>
+ ParachainHost<Block>
+ sp_block_builder::BlockBuilder<Block>
+ frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Nonce>
+ pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi<Block, Balance>
+ sp_api::Metadata<Block>
+ sp_offchain::OffchainWorkerApi<Block>
+ sp_session::SessionKeys<Block>
+ authority_discovery_primitives::AuthorityDiscoveryApi<Block>
where
<Self as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
{}
impl<Api> RuntimeApiCollection for Api
where
Api: sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
+ sp_api::ApiExt<Block, Error = sp_blockchain::Error>
+ babe_primitives::BabeApi<Block>
+ grandpa_primitives::GrandpaApi<Block>
+ ParachainHost<Block>
+ sp_block_builder::BlockBuilder<Block>
+ frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Nonce>
+ pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi<Block, Balance>
+ sp_api::Metadata<Block>
+ sp_offchain::OffchainWorkerApi<Block>
+ sp_session::SessionKeys<Block>
+ authority_discovery_primitives::AuthorityDiscoveryApi<Block>,
<Self as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
{}
/// Trait that abstracts over all available client implementations.
///
/// For a concrete type there exists [`Client`].
pub trait AbstractClient<Block, Backend>:
BlockchainEvents<Block> + Sized + Send + Sync
+ ProvideRuntimeApi<Block, Api = Runtime::RuntimeApi>
+ ProvideRuntimeApi<Block>
+ HeaderBackend<Block>
+ CallApiAt<
Block,
Error = sp_blockchain::Error,
StateBackend = Backend ::State
StateBackend = Backend::State
>
where
Block: BlockT,
Backend: BackendT<Block>,
Runtime: ConstructRuntimeApi<Block, Self>
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Self::Api: RuntimeApiCollection<StateBackend = Backend::State>,
{}
impl<Block, Backend, Runtime, Client> PolkadotClient<Block, Backend, Runtime> for Client
impl<Block, Backend, Client> AbstractClient<Block, Backend> for Client
where
Block: BlockT,
Runtime: ConstructRuntimeApi<Block, Self>,
Backend: BackendT<Block>,
Client: BlockchainEvents<Block> + ProvideRuntimeApi<Block, Api = Runtime::RuntimeApi> + HeaderBackend<Block>
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Client: BlockchainEvents<Block> + ProvideRuntimeApi<Block> + HeaderBackend<Block>
+ Sized + Send + Sync
+ CallApiAt<
Block,
Error = sp_blockchain::Error,
StateBackend = Backend ::State
>
StateBackend = Backend::State
>,
Client::Api: RuntimeApiCollection<StateBackend = Backend::State>,
{}
/// Execute something with the client instance.
///
/// As there exist multiple chains inside Polkadot, like Polkadot itself, Kusama, Westend etc,
/// there can exist different kinds of client types. As these client types differ in the generics
/// that are being used, we can not easily return them from a function. For returning them from a
/// function there exists [`Client`]. However, the problem on how to use this client instance still
/// exists. This trait "solves" it in a dirty way. It requires a type to implement this trait and
/// than the [`execute_with_client`](ExecuteWithClient::execute_with_client) function can be called
/// with any possible client instance.
///
/// In a perfect world, we could make a closure work in this way.
pub trait ExecuteWithClient {
/// The return type when calling this instance.
type Output;
/// Execute whatever should be executed with the given client instance.
fn execute_with_client<Client, Api, Backend>(self, client: Arc<Client>) -> Self::Output
where
<Api as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
Backend: sc_client_api::Backend<Block>,
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Api: crate::RuntimeApiCollection<StateBackend = Backend::State>,
Client: AbstractClient<Block, Backend, Api = Api> + 'static;
}
/// A handle to a Polkadot client instance.
///
/// The Polkadot service supports multiple different runtimes (Westend, Polkadot itself, etc). As each runtime has a
/// specialized client, we need to hide them behind a trait. This is this trait.
///
/// When wanting to work with the inner client, you need to use `execute_with`.
///
/// See [`ExecuteWithClient`](trait.ExecuteWithClient.html) for more information.
pub trait ClientHandle {
/// Execute the given something with the client.
fn execute_with<T: ExecuteWithClient>(&self, t: T) -> T::Output;
}
/// A client instance of Polkadot.
///
/// See [`ExecuteWithClient`] for more information.
#[derive(Clone)]
pub enum Client {
Polkadot(Arc<crate::FullClient<polkadot_runtime::RuntimeApi, crate::PolkadotExecutor>>),
Westend(Arc<crate::FullClient<westend_runtime::RuntimeApi, crate::WestendExecutor>>),
Kusama(Arc<crate::FullClient<kusama_runtime::RuntimeApi, crate::KusamaExecutor>>),
}
impl ClientHandle for Client {
fn execute_with<T: ExecuteWithClient>(&self, t: T) -> T::Output {
match self {
Self::Polkadot(client) => {
T::execute_with_client::<_, _, crate::FullBackend>(t, client.clone())
},
Self::Westend(client) => {
T::execute_with_client::<_, _, crate::FullBackend>(t, client.clone())
},
Self::Kusama(client) => {
T::execute_with_client::<_, _, crate::FullBackend>(t, client.clone())
},
}
}
}
impl sc_client_api::UsageProvider<Block> for Client {
fn usage_info(&self) -> sc_client_api::ClientInfo<Block> {
match self {
Self::Polkadot(client) => client.usage_info(),
Self::Westend(client) => client.usage_info(),
Self::Kusama(client) => client.usage_info(),
}
}
}
impl sc_client_api::BlockBackend<Block> for Client {
fn block_body(
&self,
id: &BlockId<Block>
) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
match self {
Self::Polkadot(client) => client.block_body(id),
Self::Westend(client) => client.block_body(id),
Self::Kusama(client) => client.block_body(id),
}
}
fn block(&self, id: &BlockId<Block>) -> sp_blockchain::Result<Option<SignedBlock<Block>>> {
match self {
Self::Polkadot(client) => client.block(id),
Self::Westend(client) => client.block(id),
Self::Kusama(client) => client.block(id),
}
}
fn block_status(&self, id: &BlockId<Block>) -> sp_blockchain::Result<BlockStatus> {
match self {
Self::Polkadot(client) => client.block_status(id),
Self::Westend(client) => client.block_status(id),
Self::Kusama(client) => client.block_status(id),
}
}
fn justification(
&self,
id: &BlockId<Block>
) -> sp_blockchain::Result<Option<Justification>> {
match self {
Self::Polkadot(client) => client.justification(id),
Self::Westend(client) => client.justification(id),
Self::Kusama(client) => client.justification(id),
}
}
fn block_hash(
&self,
number: NumberFor<Block>
) -> sp_blockchain::Result<Option<<Block as BlockT>::Hash>> {
match self {
Self::Polkadot(client) => client.block_hash(number),
Self::Westend(client) => client.block_hash(number),
Self::Kusama(client) => client.block_hash(number),
}
}
}
impl sc_client_api::StorageProvider<Block, crate::FullBackend> for Client {
fn storage(
&self,
id: &BlockId<Block>,
key: &StorageKey,
) -> sp_blockchain::Result<Option<StorageData>> {
match self {
Self::Polkadot(client) => client.storage(id, key),
Self::Westend(client) => client.storage(id, key),
Self::Kusama(client) => client.storage(id, key),
}
}
fn storage_keys(
&self,
id: &BlockId<Block>,
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<StorageKey>> {
match self {
Self::Polkadot(client) => client.storage_keys(id, key_prefix),
Self::Westend(client) => client.storage_keys(id, key_prefix),
Self::Kusama(client) => client.storage_keys(id, key_prefix),
}
}
fn storage_hash(
&self,
id: &BlockId<Block>,
key: &StorageKey,
) -> sp_blockchain::Result<Option<<Block as BlockT>::Hash>> {
match self {
Self::Polkadot(client) => client.storage_hash(id, key),
Self::Westend(client) => client.storage_hash(id, key),
Self::Kusama(client) => client.storage_hash(id, key),
}
}
fn storage_pairs(
&self,
id: &BlockId<Block>,
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<(StorageKey, StorageData)>> {
match self {
Self::Polkadot(client) => client.storage_pairs(id, key_prefix),
Self::Westend(client) => client.storage_pairs(id, key_prefix),
Self::Kusama(client) => client.storage_pairs(id, key_prefix),
}
}
fn storage_keys_iter<'a>(
&self,
id: &BlockId<Block>,
prefix: Option<&'a StorageKey>,
start_key: Option<&StorageKey>,
) -> sp_blockchain::Result<KeyIterator<'a, <crate::FullBackend as sc_client_api::Backend<Block>>::State, Block>> {
match self {
Self::Polkadot(client) => client.storage_keys_iter(id, prefix, start_key),
Self::Westend(client) => client.storage_keys_iter(id, prefix, start_key),
Self::Kusama(client) => client.storage_keys_iter(id, prefix, start_key),
}
}
fn child_storage(
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key: &StorageKey,
) -> sp_blockchain::Result<Option<StorageData>> {
match self {
Self::Polkadot(client) => client.child_storage(id, child_info, key),
Self::Westend(client) => client.child_storage(id, child_info, key),
Self::Kusama(client) => client.child_storage(id, child_info, key),
}
}
fn child_storage_keys(
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key_prefix: &StorageKey,
) -> sp_blockchain::Result<Vec<StorageKey>> {
match self {
Self::Polkadot(client) => client.child_storage_keys(id, child_info, key_prefix),
Self::Westend(client) => client.child_storage_keys(id, child_info, key_prefix),
Self::Kusama(client) => client.child_storage_keys(id, child_info, key_prefix),
}
}
fn child_storage_hash(
&self,
id: &BlockId<Block>,
child_info: &ChildInfo,
key: &StorageKey,
) -> sp_blockchain::Result<Option<<Block as BlockT>::Hash>> {
match self {
Self::Polkadot(client) => client.child_storage_hash(id, child_info, key),
Self::Westend(client) => client.child_storage_hash(id, child_info, key),
Self::Kusama(client) => client.child_storage_hash(id, child_info, key),
}
}
fn max_key_changes_range(
&self,
first: NumberFor<Block>,
last: BlockId<Block>,
) -> sp_blockchain::Result<Option<(NumberFor<Block>, BlockId<Block>)>> {
match self {
Self::Polkadot(client) => client.max_key_changes_range(first, last),
Self::Westend(client) => client.max_key_changes_range(first, last),
Self::Kusama(client) => client.max_key_changes_range(first, last),
}
}
fn key_changes(
&self,
first: NumberFor<Block>,
last: BlockId<Block>,
storage_key: Option<&PrefixedStorageKey>,
key: &StorageKey,
) -> sp_blockchain::Result<Vec<(NumberFor<Block>, u32)>> {
match self {
Self::Polkadot(client) => client.key_changes(first, last, storage_key, key),
Self::Westend(client) => client.key_changes(first, last, storage_key, key),
Self::Kusama(client) => client.key_changes(first, last, storage_key, key),
}
}
}
......@@ -22,13 +22,12 @@ mod client;
use std::sync::Arc;
use std::time::Duration;
use polkadot_primitives::v1::{AccountId, Nonce, Balance};
use service::{error::Error as ServiceError};
use service::{error::Error as ServiceError, RpcHandlers};
use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
use sc_executor::native_executor_instance;
use log::info;
use sp_blockchain::HeaderBackend;
use polkadot_overseer::{self as overseer, AllSubsystems, BlockInfo, Overseer, OverseerHandler};
use polkadot_overseer::{AllSubsystems, BlockInfo, Overseer, OverseerHandler};
use polkadot_subsystem::DummySubsystem;
use polkadot_node_core_proposer::ProposerFactory;
use sp_trie::PrefixedMemoryDB;
......@@ -55,7 +54,7 @@ pub use polkadot_runtime;
pub use kusama_runtime;
pub use westend_runtime;
use prometheus_endpoint::Registry;
pub use self::client::PolkadotClient;
pub use self::client::{AbstractClient, Client, RuntimeApiCollection};
native_executor_instance!(
pub PolkadotExecutor,
......@@ -78,40 +77,6 @@ native_executor_instance!(
frame_benchmarking::benchmarking::HostFunctions,
);
/// A set of APIs that polkadot-like runtimes must implement.
pub trait RuntimeApiCollection:
sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
+ sp_api::ApiExt<Block, Error = sp_blockchain::Error>
+ babe_primitives::BabeApi<Block>
+ grandpa_primitives::GrandpaApi<Block>
+ sp_block_builder::BlockBuilder<Block>
+ frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Nonce>
+ pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi<Block, Balance>
+ sp_api::Metadata<Block>
+ sp_offchain::OffchainWorkerApi<Block>
+ sp_session::SessionKeys<Block>
+ authority_discovery_primitives::AuthorityDiscoveryApi<Block>
where
<Self as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<HashFor<Block>>,
{}
impl<Api> RuntimeApiCollection for Api
where
Api:
sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>
+ sp_api::ApiExt<Block, Error = sp_blockchain::Error>
+ babe_primitives::BabeApi<Block>
+ grandpa_primitives::GrandpaApi<Block>
+ sp_block_builder::BlockBuilder<Block>
+ frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Nonce>
+ pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi<Block, Balance>
+ sp_api::Metadata<Block>
+ sp_offchain::OffchainWorkerApi<Block>
+ sp_session::SessionKeys<Block>
+ authority_discovery_primitives::AuthorityDiscoveryApi<Block>,
<Self as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<HashFor<Block>>,
{}
/// Can be called for a `Configuration` to check if it is a configuration for the `Kusama` network.
pub trait IdentifyVariant {
/// Returns if this is a configuration for the `Kusama` network.
......@@ -313,13 +278,14 @@ fn real_overseer<S: SpawnNamed>(
fn new_full<RuntimeApi, Executor>(
mut config: Configuration,
collating_for: Option<(CollatorId, ParaId)>,
_max_block_data_size: Option<u64>,
_authority_discovery_enabled: bool,
_slot_duration: u64,
authority_discovery_enabled: bool,
grandpa_pause: Option<(u32, u32)>,
) -> Result<(
TaskManager,
Arc<FullClient<RuntimeApi, Executor>>,
Arc<sc_network::NetworkService<Block, <Block as BlockT>::Hash>>,
RpcHandlers,
OverseerHandler,
), Error>
where
RuntimeApi: ConstructRuntimeApi<Block, FullClient<RuntimeApi, Executor>> + Send + Sync + 'static,
......@@ -337,7 +303,13 @@ fn new_full<RuntimeApi, Executor>(
let name = config.network.node_name.clone();
let service::PartialComponents {
client, backend, mut task_manager, keystore, select_chain, import_queue, transaction_pool,
client,
backend,
mut task_manager,
keystore,
select_chain,
import_queue,
transaction_pool,
inherent_data_providers,
other: (rpc_extensions_builder, import_setup, rpc_setup)
} = new_partial::<RuntimeApi, Executor>(&mut config)?;
......@@ -368,7 +340,7 @@ fn new_full<RuntimeApi, Executor>(
let telemetry_connection_sinks = service::TelemetryConnectionSinks::default();
service::spawn_tasks(service::SpawnTasksParams {
let rpc_handlers = service::spawn_tasks(service::SpawnTasksParams {
config,
backend: backend.clone(),
client: client.clone(),
......@@ -411,7 +383,7 @@ fn new_full<RuntimeApi, Executor>(
task_manager.spawn_essential_handle().spawn_blocking("overseer", Box::pin(async move {
use futures::{pin_mut, select, FutureExt};
let forward = overseer::forward_events(overseer_client, handler);
let forward = polkadot_overseer::forward_events(overseer_client, handler_clone);
let forward = forward.fuse();
let overseer_fut = overseer.run().fuse();
......@@ -435,7 +407,7 @@ fn new_full<RuntimeApi, Executor>(
let proposer = ProposerFactory::new(
client.clone(),
transaction_pool,
handler_clone,
handler.clone(),
);
let babe_config = babe::BabeParams {
......@@ -457,7 +429,7 @@ fn new_full<RuntimeApi, Executor>(
// if the node isn't actively participating in consensus then it doesn't
// need a keystore, regardless of which protocol we use below.
let keystore = if is_authority {
let keystore_opt = if is_authority {
Some(keystore.clone() as BareCryptoStorePtr)
} else {
None
......@@ -469,7 +441,7 @@ fn new_full<RuntimeApi, Executor>(
justification_period: 512,
name: Some(name),
observer_enabled: false,
keystore,
keystore: keystore_opt,
is_authority: role.is_network_authority(),
};
......@@ -508,7 +480,7 @@ fn new_full<RuntimeApi, Executor>(
inherent_data_providers: inherent_data_providers.clone(),
telemetry_on_connect: Some(telemetry_connection_sinks.on_connect_stream()),
voting_rule,
prometheus_registry,
prometheus_registry: prometheus_registry.clone(),
shared_voter_state,
};
......@@ -524,15 +496,51 @@ fn new_full<RuntimeApi, Executor>(
)?;
}
if matches!(role, Role::Authority{..} | Role::Sentry{..}) {
use sc_network::Event;
use futures::StreamExt;
if authority_discovery_enabled {
let (sentries, authority_discovery_role) = match role {
Role::Authority { ref sentry_nodes } => (
sentry_nodes.clone(),
authority_discovery::Role::Authority (
keystore.clone(),
),
),
Role::Sentry {..} => (
vec![],
authority_discovery::Role::Sentry,
),
_ => unreachable!("Due to outer matches! constraint; qed."),
};
let network_event_stream = network.event_stream("authority-discovery");
let dht_event_stream = network_event_stream.filter_map(|e| async move { match e {
Event::Dht(e) => Some(e),
_ => None,
}}).boxed();
let (authority_discovery_worker, _service) = authority_discovery::new_worker_and_service(
client.clone(),
network.clone(),
sentries,
dht_event_stream,
authority_discovery_role,
prometheus_registry.clone(),
);
task_manager.spawn_handle().spawn("authority-discovery-worker", authority_discovery_worker);
}
}
network_starter.start_network();
Ok((task_manager, client))
Ok((task_manager, client, network, rpc_handlers, handler))
}
pub struct FullNodeHandles;
/// Builds a new service for a light client.
fn new_light<Runtime, Dispatch>(mut config: Configuration) -> Result<TaskManager, Error>
fn new_light<Runtime, Dispatch>(mut config: Configuration) -> Result<(TaskManager, RpcHandlers), Error>
where
Runtime: 'static + Send + Sync + ConstructRuntimeApi<Block, LightClient<Runtime, Dispatch>>,
<Runtime as ConstructRuntimeApi<Block, LightClient<Runtime, Dispatch>>>::RuntimeApi:
......@@ -617,19 +625,25 @@ fn new_light<Runtime, Dispatch>(mut config: Configuration) -> Result<TaskManager
let rpc_extensions = polkadot_rpc::create_light(light_deps);
service::spawn_tasks(service::SpawnTasksParams {
let rpc_handlers = service::spawn_tasks(service::SpawnTasksParams {
on_demand: Some(on_demand),
remote_blockchain: Some(backend.remote_blockchain()),
rpc_extensions_builder: Box::new(service::NoopRpcExtensionBuilder(rpc_extensions)),
task_manager: &mut task_manager,
telemetry_connection_sinks: service::TelemetryConnectionSinks::default(),
config, keystore, backend, transaction_pool, client, network, network_status_sinks,
config,
keystore,
backend,
transaction_pool,
client,
network,
network_status_sinks,
system_rpc_tx,
})?;