Unverified Commit f89020da authored by Robert Klotzner's avatar Robert Klotzner Committed by GitHub
Browse files

Move non runtime related stuff into node/primitives (#2743)

* Remove stuff out of the runtime that does not belong there.

There might be more, but it is a start.

* White space fixes.

* Fix tests.

* Leave whitespace in ui tests alone.

* Add back zstd for no reason.

* Fix browser wasm (hopefully)
parent 3b4a1f47
Pipeline #131226 failed with stages
in 23 minutes and 58 seconds
......@@ -5373,6 +5373,7 @@ dependencies = [
"polkadot-erasure-coding",
"polkadot-node-core-runtime-api",
"polkadot-node-network-protocol",
"polkadot-node-primitives",
"polkadot-node-subsystem",
"polkadot-node-subsystem-test-helpers",
"polkadot-node-subsystem-util",
......@@ -5403,6 +5404,7 @@ dependencies = [
"parity-scale-codec",
"polkadot-erasure-coding",
"polkadot-node-network-protocol",
"polkadot-node-primitives",
"polkadot-node-subsystem",
"polkadot-node-subsystem-test-helpers",
"polkadot-node-subsystem-util",
......@@ -5478,6 +5480,7 @@ name = "polkadot-erasure-coding"
version = "0.8.30"
dependencies = [
"parity-scale-codec",
"polkadot-node-primitives",
"polkadot-primitives",
"reed-solomon-novelpoly",
"sp-core",
......@@ -5590,6 +5593,7 @@ dependencies = [
"parity-scale-codec",
"parking_lot 0.11.1",
"polkadot-erasure-coding",
"polkadot-node-primitives",
"polkadot-node-subsystem",
"polkadot-node-subsystem-test-helpers",
"polkadot-node-subsystem-util",
......@@ -5759,6 +5763,7 @@ dependencies = [
"mick-jaeger",
"parity-scale-codec",
"parking_lot 0.11.1",
"polkadot-node-primitives",
"polkadot-primitives",
"sc-network",
"sp-core",
......@@ -5785,15 +5790,18 @@ version = "0.1.0"
dependencies = [
"futures 0.3.13",
"parity-scale-codec",
"polkadot-parachain",
"polkadot-primitives",
"polkadot-statement-table",
"schnorrkel",
"serde",
"sp-application-crypto",
"sp-consensus-babe",
"sp-consensus-vrf",
"sp-core",
"sp-runtime",
"thiserror",
"zstd",
]
[[package]]
......@@ -5953,7 +5961,6 @@ dependencies = [
"sp-trie",
"sp-version",
"thiserror",
"zstd",
]
[[package]]
......@@ -6207,6 +6214,7 @@ dependencies = [
"polkadot-node-core-proposer",
"polkadot-node-core-provisioner",
"polkadot-node-core-runtime-api",
"polkadot-node-primitives",
"polkadot-node-subsystem",
"polkadot-node-subsystem-util",
"polkadot-overseer",
......
......@@ -72,9 +72,9 @@ impl std::fmt::Display for CandidateHash {
}
impl sp_std::fmt::Debug for CandidateHash {
fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result {
write!(f, "{:?}", self.0)
}
fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result {
write!(f, "{:?}", self.0)
}
}
/// Index of a transaction in the relay chain. 32-bit should be plenty.
......
......@@ -5,7 +5,8 @@ authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
primitives = { package = "polkadot-primitives", path = "../primitives" }
polkadot-primitives = { package = "polkadot-primitives", path = "../primitives" }
polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../node/primitives" }
novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" }
parity-scale-codec = { version = "2.0.0", default-features = false, features = ["std", "derive"] }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
......
......@@ -25,8 +25,8 @@
//! The data is coded so any f+1 chunks can be used to reconstruct the full data.
use parity_scale_codec::{Encode, Decode};
use primitives::v0::{self, Hash as H256, BlakeTwo256, HashT};
use primitives::v1;
use polkadot_primitives::v0::{self, Hash as H256, BlakeTwo256, HashT};
use polkadot_node_primitives::AvailableData;
use sp_core::Blake2Hasher;
use trie::{EMPTY_PREFIX, MemoryDB, Trie, TrieMut, trie_types::{TrieDBMut, TrieDB}};
use thiserror::Error;
......@@ -122,7 +122,7 @@ pub fn obtain_chunks_v0(n_validators: usize, data: &v0::AvailableData)
/// Obtain erasure-coded chunks for v1 `AvailableData`, one for each validator.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn obtain_chunks_v1(n_validators: usize, data: &v1::AvailableData)
pub fn obtain_chunks_v1(n_validators: usize, data: &AvailableData)
-> Result<Vec<Vec<u8>>, Error>
{
obtain_chunks(n_validators, data)
......@@ -169,7 +169,7 @@ pub fn reconstruct_v0<'a, I: 'a>(n_validators: usize, chunks: I)
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn reconstruct_v1<'a, I: 'a>(n_validators: usize, chunks: I)
-> Result<v1::AvailableData, Error>
-> Result<AvailableData, Error>
where I: IntoIterator<Item=(&'a [u8], usize)>
{
reconstruct(n_validators, chunks)
......@@ -368,14 +368,14 @@ impl<'a, I: Iterator<Item=&'a [u8]>> parity_scale_codec::Input for ShardInput<'a
#[cfg(test)]
mod tests {
use super::*;
use primitives::v0::{AvailableData, BlockData, PoVBlock};
use polkadot_primitives::v0::{AvailableData, BlockData, PoVBlock};
#[test]
fn field_order_is_right_size() {
assert_eq!(MAX_VALIDATORS, 65536);
}
#[test]
#[test]
fn round_trip_works() {
let pov_block = PoVBlock {
block_data: BlockData((0..255).collect()),
......
......@@ -26,7 +26,7 @@ use futures::{
sink::SinkExt,
stream::StreamExt,
};
use polkadot_node_primitives::CollationGenerationConfig;
use polkadot_node_primitives::{CollationGenerationConfig, AvailableData, PoV};
use polkadot_node_subsystem::{
messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage},
FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemResult,
......@@ -37,9 +37,9 @@ use polkadot_node_subsystem_util::{
metrics::{self, prometheus},
};
use polkadot_primitives::v1::{
collator_signature_payload, AvailableData, CandidateCommitments,
collator_signature_payload, CandidateCommitments,
CandidateDescriptor, CandidateReceipt, CoreState, Hash, OccupiedCoreAssumption,
PersistedValidationData, PoV,
PersistedValidationData,
};
use sp_core::crypto::Pair;
use std::sync::Arc;
......@@ -465,7 +465,7 @@ mod tests {
task::{Context as FuturesContext, Poll},
Future,
};
use polkadot_node_primitives::{Collation, CollationResult};
use polkadot_node_primitives::{Collation, CollationResult, BlockData, PoV};
use polkadot_node_subsystem::messages::{
AllMessages, RuntimeApiMessage, RuntimeApiRequest,
};
......@@ -473,8 +473,8 @@ mod tests {
subsystem_test_harness, TestSubsystemContextHandle,
};
use polkadot_primitives::v1::{
BlockData, BlockNumber, CollatorPair, Id as ParaId,
PersistedValidationData, PoV, ScheduledCore,
BlockNumber, CollatorPair, Id as ParaId,
PersistedValidationData, ScheduledCore,
};
use std::pin::Pin;
......
......@@ -52,7 +52,7 @@ pub enum RequiredTranches {
needed: DelayTranche,
/// The amount of missing votes that should be tolerated.
tolerated_missing: usize,
/// When the next no-show would be, if any. This is used to schedule the next wakeup in the
/// When the next no-show would be, if any. This is used to schedule the next wakeup in the
/// event that there are some assignments that don't have corresponding approval votes. If this
/// is `None`, all assignments have approvals.
next_no_show: Option<Tick>,
......
......@@ -37,10 +37,10 @@ use polkadot_node_subsystem_util::{
use polkadot_primitives::v1::{
ValidatorIndex, Hash, SessionIndex, SessionInfo, CandidateHash,
CandidateReceipt, BlockNumber, PersistedValidationData,
ValidationCode, CandidateDescriptor, PoV, ValidatorPair, ValidatorSignature, ValidatorId,
ValidationCode, CandidateDescriptor, ValidatorPair, ValidatorSignature, ValidatorId,
CandidateIndex, GroupIndex,
};
use polkadot_node_primitives::ValidationResult;
use polkadot_node_primitives::{ValidationResult, PoV};
use polkadot_node_primitives::approval::{
IndirectAssignmentCert, IndirectSignedApprovalVote, ApprovalVote, DelayTranche,
};
......
......@@ -19,6 +19,7 @@ polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsys
polkadot-node-subsystem-util = { path = "../../subsystem-util" }
polkadot-overseer = { path = "../../overseer" }
polkadot-primitives = { path = "../../../primitives" }
polkadot-node-primitives = { path = "../../primitives" }
sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
......
......@@ -32,9 +32,12 @@ use kvdb_rocksdb::{Database, DatabaseConfig};
use kvdb::{KeyValueDB, DBTransaction};
use polkadot_primitives::v1::{
Hash, AvailableData, BlockNumber, CandidateEvent, ErasureChunk, ValidatorIndex, CandidateHash,
Hash, BlockNumber, CandidateEvent, ValidatorIndex, CandidateHash,
CandidateReceipt,
};
use polkadot_node_primitives::{
ErasureChunk, AvailableData,
};
use polkadot_subsystem::{
FromOverseer, OverseerSignal, SubsystemError, Subsystem, SubsystemContext, SpawnedSubsystem,
ActiveLeavesUpdate,
......
......@@ -25,10 +25,11 @@ use futures::{
};
use polkadot_primitives::v1::{
AvailableData, BlockData, CandidateDescriptor, CandidateReceipt, HeadData,
PersistedValidationData, PoV, Id as ParaId, CandidateHash, Header, ValidatorId,
CandidateDescriptor, CandidateReceipt, HeadData,
PersistedValidationData, Id as ParaId, CandidateHash, Header, ValidatorId,
CoreIndex, GroupIndex,
};
use polkadot_node_primitives::{AvailableData, BlockData, PoV};
use polkadot_node_subsystem_util::TimeoutExt;
use polkadot_subsystem::{
ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, messages::AllMessages, ActivatedLeaf,
......
......@@ -27,12 +27,12 @@ use futures::{channel::{mpsc, oneshot}, Future, FutureExt, SinkExt, StreamExt};
use sp_keystore::SyncCryptoStorePtr;
use polkadot_primitives::v1::{
AvailableData, BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash,
BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash,
CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreIndex, CoreState, Hash, Id as ParaId,
PoV, SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation,
SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation,
};
use polkadot_node_primitives::{
Statement, SignedFullStatement, ValidationResult,
Statement, SignedFullStatement, ValidationResult, PoV, AvailableData,
};
use polkadot_subsystem::{
PerLeafSpan, Stage, SubsystemSender,
......@@ -1348,12 +1348,12 @@ mod tests {
use super::*;
use assert_matches::assert_matches;
use futures::{future, Future};
use polkadot_primitives::v1::{BlockData, GroupRotationInfo, HeadData, PersistedValidationData, ScheduledCore};
use polkadot_primitives::v1::{GroupRotationInfo, HeadData, PersistedValidationData, ScheduledCore};
use polkadot_subsystem::{
messages::{RuntimeApiRequest, RuntimeApiMessage},
ActiveLeavesUpdate, FromOverseer, OverseerSignal, ActivatedLeaf,
};
use polkadot_node_primitives::InvalidCandidate;
use polkadot_node_primitives::{InvalidCandidate, BlockData};
use sp_keyring::Sr25519Keyring;
use sp_application_crypto::AppKey;
use sp_keystore::{CryptoStore, SyncCryptoStore};
......@@ -2763,7 +2763,7 @@ mod tests {
virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await;
// Not deterministic which message comes first:
for _ in 0..2 {
for _ in 0u32..2 {
match virtual_overseer.recv().await {
AllMessages::Provisioner(
ProvisionerMessage::ProvisionableData(
......
......@@ -37,9 +37,9 @@ use polkadot_node_subsystem_util::{
JobTrait, JobSender, Validator, metrics::{self, prometheus},
};
use polkadot_primitives::v1::{
CandidateReceipt, CollatorId, CoreState, CoreIndex, Hash, Id as ParaId, PoV, BlockNumber,
CandidateReceipt, CollatorId, CoreState, CoreIndex, Hash, Id as ParaId, BlockNumber,
};
use polkadot_node_primitives::SignedFullStatement;
use polkadot_node_primitives::{SignedFullStatement, PoV};
use std::{pin::Pin, sync::Arc};
use thiserror::Error;
......@@ -510,7 +510,7 @@ pub type CandidateSelectionSubsystem<Spawner> = JobSubsystem<CandidateSelectionJ
mod tests {
use super::*;
use futures::lock::Mutex;
use polkadot_primitives::v1::BlockData;
use polkadot_node_primitives::BlockData;
use polkadot_node_subsystem::messages::AllMessages;
use sp_core::crypto::Public;
use std::sync::Arc;
......
......@@ -33,9 +33,9 @@ use polkadot_subsystem::{
};
use polkadot_node_subsystem_util::metrics::{self, prometheus};
use polkadot_subsystem::errors::RuntimeApiError;
use polkadot_node_primitives::{ValidationResult, InvalidCandidate};
use polkadot_node_primitives::{ValidationResult, InvalidCandidate, PoV};
use polkadot_primitives::v1::{
ValidationCode, PoV, CandidateDescriptor, PersistedValidationData,
ValidationCode, CandidateDescriptor, PersistedValidationData,
OccupiedCoreAssumption, Hash, CandidateCommitments,
};
use polkadot_parachain::wasm_executor::{
......@@ -568,7 +568,8 @@ impl metrics::Metrics for Metrics {
mod tests {
use super::*;
use polkadot_node_subsystem_test_helpers as test_helpers;
use polkadot_primitives::v1::{HeadData, BlockData, UpwardMessage};
use polkadot_primitives::v1::{HeadData, UpwardMessage};
use polkadot_node_primitives::BlockData;
use sp_core::testing::TaskExecutor;
use futures::executor;
use assert_matches::assert_matches;
......
......@@ -11,6 +11,7 @@ mick-jaeger = "0.1.4"
lazy_static = "1.4"
parking_lot = "0.11.1"
polkadot-primitives = { path = "../../primitives" }
polkadot-node-primitives = { path = "../primitives" }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
thiserror = "1.0.23"
......
......@@ -84,7 +84,8 @@
//! ```
use parity_scale_codec::Encode;
use polkadot_primitives::v1::{BlakeTwo256, CandidateHash, Hash, HashT, Id as ParaId, PoV, ValidatorIndex};
use polkadot_primitives::v1::{BlakeTwo256, CandidateHash, Hash, HashT, Id as ParaId, ValidatorIndex};
use polkadot_node_primitives::PoV;
use sc_network::PeerId;
use std::fmt;
......
......@@ -91,9 +91,9 @@ impl<T> MeteredReceiver<T> {
}
impl<T> futures::stream::FusedStream for MeteredReceiver<T> {
fn is_terminated(&self) -> bool {
self.inner.is_terminated()
}
fn is_terminated(&self) -> bool {
self.inner.is_terminated()
}
}
......@@ -155,32 +155,32 @@ impl<T> MeteredSender<T> {
}
impl<T> futures::sink::Sink<T> for MeteredSender<T> {
type Error = mpsc::SendError;
fn start_send(mut self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
Pin::new(&mut self.inner).start_send(item)
}
fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Pin::new(&mut self.inner).poll_ready(cx)
}
fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match Pin::new(&mut self.inner).poll_close(cx) {
val @ Poll::Ready(_)=> {
val
}
other => other,
}
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match Pin::new(&mut self.inner).poll_flush(cx) {
val @ Poll::Ready(_)=> {
self.meter.note_sent();
val
}
other => other,
}
}
type Error = mpsc::SendError;
fn start_send(mut self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
Pin::new(&mut self.inner).start_send(item)
}
fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Pin::new(&mut self.inner).poll_ready(cx)
}
fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match Pin::new(&mut self.inner).poll_close(cx) {
val @ Poll::Ready(_)=> {
val
}
other => other,
}
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match Pin::new(&mut self.inner).poll_flush(cx) {
val @ Poll::Ready(_)=> {
self.meter.note_sent();
val
}
other => other,
}
}
}
......@@ -26,74 +26,74 @@ use super::Meter;
/// Create a wrapped `mpsc::channel` pair of `MeteredSender` and `MeteredReceiver`.
pub fn unbounded<T>() -> (UnboundedMeteredSender<T>, UnboundedMeteredReceiver<T>) {
let (tx, rx) = mpsc::unbounded();
let shared_meter = Meter::default();
let tx = UnboundedMeteredSender { meter: shared_meter.clone(), inner: tx };
let rx = UnboundedMeteredReceiver { meter: shared_meter, inner: rx };
(tx, rx)
let (tx, rx) = mpsc::unbounded();
let shared_meter = Meter::default();
let tx = UnboundedMeteredSender { meter: shared_meter.clone(), inner: tx };
let rx = UnboundedMeteredReceiver { meter: shared_meter, inner: rx };
(tx, rx)
}
/// A receiver tracking the messages consumed by itself.
#[derive(Debug)]
pub struct UnboundedMeteredReceiver<T> {
// count currently contained messages
meter: Meter,
inner: mpsc::UnboundedReceiver<T>,
// count currently contained messages
meter: Meter,
inner: mpsc::UnboundedReceiver<T>,
}
impl<T> std::ops::Deref for UnboundedMeteredReceiver<T> {
type Target = mpsc::UnboundedReceiver<T>;
fn deref(&self) -> &Self::Target {
&self.inner
}
type Target = mpsc::UnboundedReceiver<T>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<T> std::ops::DerefMut for UnboundedMeteredReceiver<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
impl<T> Stream for UnboundedMeteredReceiver<T> {
type Item = T;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match mpsc::UnboundedReceiver::poll_next(Pin::new(&mut self.inner), cx) {
Poll::Ready(x) => {
self.meter.note_received();
Poll::Ready(x)
}
other => other,
}
}
/// Don't rely on the unreliable size hint.
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
type Item = T;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match mpsc::UnboundedReceiver::poll_next(Pin::new(&mut self.inner), cx) {
Poll::Ready(x) => {
self.meter.note_received();
Poll::Ready(x)
}
other => other,
}
}
/// Don't rely on the unreliable size hint.
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
impl<T> UnboundedMeteredReceiver<T> {
/// Get an updated accessor object for all metrics collected.
pub fn meter(&self) -> &Meter {
&self.meter
}
/// Attempt to receive the next item.
pub fn try_next(&mut self) -> Result<Option<T>, mpsc::TryRecvError> {
match self.inner.try_next()? {
Some(x) => {
self.meter.note_received();
Ok(Some(x))
}
None => Ok(None),
}
}
/// Get an updated accessor object for all metrics collected.
pub fn meter(&self) -> &Meter {
&self.meter
}
/// Attempt to receive the next item.
pub fn try_next(&mut self) -> Result<Option<T>, mpsc::TryRecvError> {
match self.inner.try_next()? {
Some(x) => {
self.meter.note_received();
Ok(Some(x))
}
None => Ok(None),
}
}
}
impl<T> futures::stream::FusedStream for UnboundedMeteredReceiver<T> {
fn is_terminated(&self) -> bool {
self.inner.is_terminated()
}
fn is_terminated(&self) -> bool {
self.inner.is_terminated()
}
}
......@@ -101,8 +101,8 @@ impl<T> futures::stream::FusedStream for UnboundedMeteredReceiver<T> {
/// sent across it.
#[derive(Debug)]
pub struct UnboundedMeteredSender<T> {
meter: Meter,
inner: mpsc::UnboundedSender<T>,
meter: Meter,
inner: mpsc::UnboundedSender<T>,
}
impl<T> Clone for UnboundedMeteredSender<T> {
......@@ -112,76 +112,76 @@ impl<T> Clone for UnboundedMeteredSender<T> {
}
impl<T> std::ops::Deref for UnboundedMeteredSender<T> {
type Target = mpsc::UnboundedSender<T>;
fn deref(&self) -> &Self::Target {
&self.inner
}
type Target = mpsc::UnboundedSender<T>;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<T> std::ops::DerefMut for UnboundedMeteredSender<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
impl<T> UnboundedMeteredSender<T> {
/// Get an updated accessor object for all metrics collected.
pub fn meter(&self) -> &Meter {
&self.meter
}
/// Send message, wait until capacity is available.
pub async fn send(&mut self, item: T) -> result::Result<(), mpsc::SendError>
where
Self: Unpin,
{
self.meter.note_sent();
let fut = self.inner.send(item);
futures::pin_mut!(fut);
fut.await.map_err(|e| {
/// Get an updated accessor object for all metrics collected.