From 98de97e1d956c132292d5753a36fa0d1baf2ee8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= <tomusdrw@users.noreply.github.com>
Date: Sun, 19 May 2019 19:02:09 +0200
Subject: [PATCH] Access child storage over RPC. (#2586)
* Access child storage over RPC.
* Address review grumbles.
* Test happy case in child_storage rpc.
* Remove stray printlns
* Fix line widths.
* Bump runtime again.
* Fix genesis storage root calculation for light clients.
* Don't pass values to full_storage_root child_delta.
---
substrate/core/client/db/src/storage_cache.rs | 6 +-
substrate/core/client/src/client.rs | 31 ++++++-
substrate/core/client/src/light/backend.rs | 15 +++-
substrate/core/rpc/src/state/mod.rs | 87 ++++++++++++++++++-
substrate/core/rpc/src/state/tests.rs | 44 +++++++++-
substrate/core/sr-primitives/src/lib.rs | 22 ++++-
substrate/core/state-machine/src/backend.rs | 44 ++++++++--
.../core/state-machine/src/proving_backend.rs | 6 +-
.../core/state-machine/src/trie_backend.rs | 2 +-
substrate/core/test-client/src/lib.rs | 14 ++-
.../node-template/runtime/wasm/Cargo.lock | 1 +
11 files changed, 248 insertions(+), 24 deletions(-)
diff --git a/substrate/core/client/db/src/storage_cache.rs b/substrate/core/client/db/src/storage_cache.rs
index 439a749c853..bc0a179cf84 100644
--- a/substrate/core/client/db/src/storage_cache.rs
+++ b/substrate/core/client/db/src/storage_cache.rs
@@ -397,10 +397,14 @@ impl<H: Hasher, S: StateBackend<H>, B:Block> StateBackend<H> for CachingState<H,
self.state.pairs()
}
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.state.keys(prefix)
}
+ fn child_keys(&self, child_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+ self.state.child_keys(child_key, prefix)
+ }
+
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
self.state.try_into_trie_backend()
}
diff --git a/substrate/core/client/src/client.rs b/substrate/core/client/src/client.rs
index 4091e23b5f0..257f120d72c 100644
--- a/substrate/core/client/src/client.rs
+++ b/substrate/core/client/src/client.rs
@@ -337,19 +337,46 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
self.import_lock.clone()
}
- /// Return storage entry keys in state in a block of given hash with given prefix.
+ /// Given a `BlockId` and a key prefix, return the matching child storage keys in that block.
pub fn storage_keys(&self, id: &BlockId<Block>, key_prefix: &StorageKey) -> error::Result<Vec<StorageKey>> {
let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect();
Ok(keys)
}
- /// Return single storage entry of contract under given address in state in a block of given hash.
+ /// Given a `BlockId` and a key, return the value under the key in that block.
pub fn storage(&self, id: &BlockId<Block>, key: &StorageKey) -> error::Result<Option<StorageData>> {
Ok(self.state_at(id)?
.storage(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
.map(StorageData))
}
+ /// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys.
+ pub fn child_storage_keys(
+ &self,
+ id: &BlockId<Block>,
+ child_storage_key: &StorageKey,
+ key_prefix: &StorageKey
+ ) -> error::Result<Vec<StorageKey>> {
+ let keys = self.state_at(id)?
+ .child_keys(&child_storage_key.0, &key_prefix.0)
+ .into_iter()
+ .map(StorageKey)
+ .collect();
+ Ok(keys)
+ }
+
+ /// Given a `BlockId`, a key and a child storage key, return the value under the key in that block.
+ pub fn child_storage(
+ &self,
+ id: &BlockId<Block>,
+ child_storage_key: &StorageKey,
+ key: &StorageKey
+ ) -> error::Result<Option<StorageData>> {
+ Ok(self.state_at(id)?
+ .child_storage(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
+ .map(StorageData))
+ }
+
/// Get the code at a given block.
pub fn code_at(&self, id: &BlockId<Block>) -> error::Result<Vec<u8>> {
Ok(self.storage(id, &StorageKey(well_known_keys::CODE.to_vec()))?
diff --git a/substrate/core/client/src/light/backend.rs b/substrate/core/client/src/light/backend.rs
index 52cdb6a626a..0579c8c8d4f 100644
--- a/substrate/core/client/src/light/backend.rs
+++ b/substrate/core/client/src/light/backend.rs
@@ -278,11 +278,20 @@ where
// this is only called when genesis block is imported => shouldn't be performance bottleneck
let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
storage.insert(None, top);
+
+ // create a list of children keys to re-compute roots for
+ let child_delta = children.keys()
+ .cloned()
+ .map(|storage_key| (storage_key, None))
+ .collect::<Vec<_>>();
+
+ // make sure to persist the child storage
for (child_key, child_storage) in children {
storage.insert(Some(child_key), child_storage);
}
+
let storage_update: InMemoryState<H> = storage.into();
- let (storage_root, _) = storage_update.storage_root(::std::iter::empty());
+ let (storage_root, _) = storage_update.full_storage_root(::std::iter::empty(), child_delta);
self.storage_update = Some(storage_update);
Ok(storage_root)
@@ -373,7 +382,7 @@ where
Vec::new()
}
- fn keys(&self, _prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+ fn keys(&self, _prefix: &[u8]) -> Vec<Vec<u8>> {
// whole state is not available on light node
Vec::new()
}
@@ -465,7 +474,7 @@ where
}
}
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
match *self {
OnDemandOrGenesisState::OnDemand(ref state) =>
StateBackend::<H>::keys(state, prefix),
diff --git a/substrate/core/rpc/src/state/mod.rs b/substrate/core/rpc/src/state/mod.rs
index 168c0bd6927..4cb7a3fbd89 100644
--- a/substrate/core/rpc/src/state/mod.rs
+++ b/substrate/core/rpc/src/state/mod.rs
@@ -57,7 +57,7 @@ pub trait StateApi<Hash> {
/// Returns the keys with prefix, leave empty to get all the keys
#[rpc(name = "state_getKeys")]
- fn storage_keys(&self, key: StorageKey, hash: Option<Hash>) -> Result<Vec<StorageKey>>;
+ fn storage_keys(&self, prefix: StorageKey, hash: Option<Hash>) -> Result<Vec<StorageKey>>;
/// Returns a storage entry at a specific block's state.
#[rpc(name = "state_getStorage", alias("state_getStorageAt"))]
@@ -71,6 +71,40 @@ pub trait StateApi<Hash> {
#[rpc(name = "state_getStorageSize", alias("state_getStorageSizeAt"))]
fn storage_size(&self, key: StorageKey, hash: Option<Hash>) -> Result<Option<u64>>;
+ /// Returns the keys with prefix from a child storage, leave empty to get all the keys
+ #[rpc(name = "state_getChildKeys")]
+ fn child_storage_keys(
+ &self,
+ child_storage_key: StorageKey,
+ prefix: StorageKey,
+ hash: Option<Hash>
+ ) -> Result<Vec<StorageKey>>;
+
+ /// Returns a child storage entry at a specific block's state.
+ #[rpc(name = "state_getChildStorage")]
+ fn child_storage(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey, hash: Option<Hash>
+ ) -> Result<Option<StorageData>>;
+
+ /// Returns the hash of a child storage entry at a block's state.
+ #[rpc(name = "state_getChildStorageHash")]
+ fn child_storage_hash(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey, hash: Option<Hash>
+ ) -> Result<Option<Hash>>;
+
+ /// Returns the size of a child storage entry at a block's state.
+ #[rpc(name = "state_getChildStorageSize")]
+ fn child_storage_size(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey,
+ hash: Option<Hash>
+ ) -> Result<Option<u64>>;
+
/// Returns the runtime metadata as an opaque blob.
#[rpc(name = "state_getMetadata")]
fn metadata(&self, hash: Option<Hash>) -> Result<Bytes>;
@@ -84,7 +118,12 @@ pub trait StateApi<Hash> {
/// NOTE This first returned result contains the initial state of storage for all keys.
/// Subsequent values in the vector represent changes to the previous state (diffs).
#[rpc(name = "state_queryStorage")]
- fn query_storage(&self, keys: Vec<StorageKey>, block: Hash, hash: Option<Hash>) -> Result<Vec<StorageChangeSet<Hash>>>;
+ fn query_storage(
+ &self,
+ keys: Vec<StorageKey>,
+ block: Hash,
+ hash: Option<Hash>
+ ) -> Result<Vec<StorageChangeSet<Hash>>>;
/// New runtime version subscription
#[pubsub(
@@ -324,6 +363,50 @@ impl<B, E, Block, RA> StateApi<Block::Hash> for State<B, E, Block, RA> where
Ok(self.storage(key, block)?.map(|x| x.0.len() as u64))
}
+ fn child_storage(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey,
+ block: Option<Block::Hash>
+ ) -> Result<Option<StorageData>> {
+ let block = self.unwrap_or_best(block)?;
+ trace!(target: "rpc", "Querying child storage at {:?} for key {}", block, HexDisplay::from(&key.0));
+ Ok(self.client.child_storage(&BlockId::Hash(block), &child_storage_key, &key)?)
+ }
+
+ fn child_storage_keys(
+ &self,
+ child_storage_key: StorageKey,
+ key_prefix: StorageKey,
+ block: Option<Block::Hash>
+ ) -> Result<Vec<StorageKey>> {
+ let block = self.unwrap_or_best(block)?;
+ trace!(target: "rpc", "Querying child storage keys at {:?}", block);
+ Ok(self.client.child_storage_keys(&BlockId::Hash(block), &child_storage_key, &key_prefix)?)
+ }
+
+ fn child_storage_hash(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey,
+ block: Option<Block::Hash>
+ ) -> Result<Option<Block::Hash>> {
+ use runtime_primitives::traits::{Hash, Header as HeaderT};
+ Ok(
+ self.child_storage(child_storage_key, key, block)?
+ .map(|x| <Block::Header as HeaderT>::Hashing::hash(&x.0))
+ )
+ }
+
+ fn child_storage_size(
+ &self,
+ child_storage_key: StorageKey,
+ key: StorageKey,
+ block: Option<Block::Hash>
+ ) -> Result<Option<u64>> {
+ Ok(self.child_storage(child_storage_key, key, block)?.map(|x| x.0.len() as u64))
+ }
+
fn metadata(&self, block: Option<Block::Hash>) -> Result<Bytes> {
let block = self.unwrap_or_best(block)?;
self.client.runtime_api().metadata(&BlockId::Hash(block)).map(Into::into).map_err(Into::into)
diff --git a/substrate/core/rpc/src/state/tests.rs b/substrate/core/rpc/src/state/tests.rs
index 5321116c952..73a52fbc625 100644
--- a/substrate/core/rpc/src/state/tests.rs
+++ b/substrate/core/rpc/src/state/tests.rs
@@ -17,9 +17,10 @@
use super::*;
use self::error::{Error, ErrorKind};
-use sr_io::blake2_256;
use assert_matches::assert_matches;
use consensus::BlockOrigin;
+use primitives::storage::well_known_keys;
+use sr_io::blake2_256;
use test_client::{self, runtime, AccountKeyring, TestClient, BlockBuilderExt};
#[test]
@@ -28,11 +29,46 @@ fn should_return_storage() {
let client = Arc::new(test_client::new());
let genesis_hash = client.genesis_hash();
let client = State::new(client, Subscriptions::new(core.executor()));
+ let key = StorageKey(b":code".to_vec());
+ assert!(
+ client.storage(key.clone(), Some(genesis_hash).into())
+ .map(|x| x.map(|x| x.0.len())).unwrap().unwrap()
+ > 195_000
+ );
assert_matches!(
- client.storage(StorageKey(vec![10]), Some(genesis_hash).into()),
- Ok(None)
- )
+ client.storage_hash(key.clone(), Some(genesis_hash).into()).map(|x| x.is_some()),
+ Ok(true)
+ );
+ assert!(
+ client.storage_size(key.clone(), None).unwrap().unwrap()
+ > 195_000
+ );
+}
+
+#[test]
+fn should_return_child_storage() {
+ let core = ::tokio::runtime::Runtime::new().unwrap();
+ let client = Arc::new(test_client::new());
+ let genesis_hash = client.genesis_hash();
+ let client = State::new(client, Subscriptions::new(core.executor()));
+ let child_key = StorageKey(well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect());
+ let key = StorageKey(b"key".to_vec());
+
+
+ assert_matches!(
+ client.child_storage(child_key.clone(), key.clone(), Some(genesis_hash).into()),
+ Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1
+ );
+ assert_matches!(
+ client.child_storage_hash(child_key.clone(), key.clone(), Some(genesis_hash).into())
+ .map(|x| x.is_some()),
+ Ok(true)
+ );
+ assert_matches!(
+ client.child_storage_size(child_key.clone(), key.clone(), None),
+ Ok(Some(1))
+ );
}
#[test]
diff --git a/substrate/core/sr-primitives/src/lib.rs b/substrate/core/sr-primitives/src/lib.rs
index 65a4f3d8229..bdecfe89818 100644
--- a/substrate/core/sr-primitives/src/lib.rs
+++ b/substrate/core/sr-primitives/src/lib.rs
@@ -102,12 +102,32 @@ impl BuildStorage for StorageOverlay {
fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
Ok((self, Default::default()))
}
- fn assimilate_storage(self, storage: &mut StorageOverlay, _child_storage: &mut ChildrenStorageOverlay) -> Result<(), String> {
+ fn assimilate_storage(
+ self,
+ storage: &mut StorageOverlay,
+ _child_storage: &mut ChildrenStorageOverlay
+ ) -> Result<(), String> {
storage.extend(self);
Ok(())
}
}
+#[cfg(feature = "std")]
+impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) {
+ fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
+ Ok(self)
+ }
+ fn assimilate_storage(
+ self,
+ storage: &mut StorageOverlay,
+ child_storage: &mut ChildrenStorageOverlay
+ )-> Result<(), String> {
+ storage.extend(self.0);
+ child_storage.extend(self.1);
+ Ok(())
+ }
+}
+
/// Consensus engine unique ID.
pub type ConsensusEngineId = [u8; 4];
diff --git a/substrate/core/state-machine/src/backend.rs b/substrate/core/state-machine/src/backend.rs
index 895a805e436..fd143a553eb 100644
--- a/substrate/core/state-machine/src/backend.rs
+++ b/substrate/core/state-machine/src/backend.rs
@@ -88,7 +88,22 @@ pub trait Backend<H: Hasher> {
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
/// Get all keys with given prefix
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>>;
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+ let mut all = Vec::new();
+ self.for_keys_with_prefix(prefix, |k| all.push(k.to_vec()));
+ all
+ }
+
+ /// Get all keys of child storage with given prefix
+ fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+ let mut all = Vec::new();
+ self.for_keys_in_child_storage(child_storage_key, |k| {
+ if k.starts_with(prefix) {
+ all.push(k.to_vec());
+ }
+ });
+ all
+ }
/// Try convert into trie backend.
fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>>;
@@ -286,7 +301,9 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
<H as Hasher>::Out: Ord,
{
- let existing_pairs = self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+ let existing_pairs = self.inner.get(&None)
+ .into_iter()
+ .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
@@ -307,7 +324,9 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
{
let storage_key = storage_key.to_vec();
- let existing_pairs = self.inner.get(&Some(storage_key.clone())).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+ let existing_pairs = self.inner.get(&Some(storage_key.clone()))
+ .into_iter()
+ .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
let transaction: Vec<_> = delta.into_iter().collect();
let root = child_trie_root::<H, _, _, _>(
@@ -326,11 +345,24 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
- self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))).collect()
+ self.inner.get(&None)
+ .into_iter()
+ .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
+ .collect()
}
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
- self.inner.get(&None).into_iter().flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()).collect()
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+ self.inner.get(&None)
+ .into_iter()
+ .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+ .collect()
+ }
+
+ fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+ self.inner.get(&Some(storage_key.to_vec()))
+ .into_iter()
+ .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+ .collect()
}
fn try_into_trie_backend(
diff --git a/substrate/core/state-machine/src/proving_backend.rs b/substrate/core/state-machine/src/proving_backend.rs
index fa7e94f78f2..c23838bc215 100644
--- a/substrate/core/state-machine/src/proving_backend.rs
+++ b/substrate/core/state-machine/src/proving_backend.rs
@@ -162,10 +162,14 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
self.backend.pairs()
}
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.backend.keys(prefix)
}
+ fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+ self.backend.child_keys(child_storage_key, prefix)
+ }
+
fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
{
diff --git a/substrate/core/state-machine/src/trie_backend.rs b/substrate/core/state-machine/src/trie_backend.rs
index 1ce915d5f50..00c0aca0062 100644
--- a/substrate/core/state-machine/src/trie_backend.rs
+++ b/substrate/core/state-machine/src/trie_backend.rs
@@ -105,7 +105,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
}
}
- fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+ fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
let mut read_overlay = S::Overlay::default();
let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
diff --git a/substrate/core/test-client/src/lib.rs b/substrate/core/test-client/src/lib.rs
index 770b3de323f..5154fb0c1db 100644
--- a/substrate/core/test-client/src/lib.rs
+++ b/substrate/core/test-client/src/lib.rs
@@ -34,7 +34,8 @@ pub use keyring::{sr25519::Keyring as AuthorityKeyring, AccountKeyring};
use std::{sync::Arc, collections::HashMap};
use futures::future::FutureResult;
use primitives::Blake2Hasher;
-use runtime_primitives::StorageOverlay;
+use primitives::storage::well_known_keys;
+use runtime_primitives::{StorageOverlay, ChildrenStorageOverlay};
use runtime_primitives::traits::{
Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor
};
@@ -271,7 +272,7 @@ fn genesis_config(support_changes_trie: bool) -> GenesisConfig {
fn genesis_storage(
support_changes_trie: bool,
extension: HashMap<Vec<u8>, Vec<u8>>
-) -> StorageOverlay {
+) -> (StorageOverlay, ChildrenStorageOverlay) {
let mut storage = genesis_config(support_changes_trie).genesis_map();
storage.extend(extension.into_iter());
@@ -280,7 +281,14 @@ fn genesis_storage(
);
let block: runtime::Block = client::genesis::construct_genesis_block(state_root);
storage.extend(additional_storage_with_genesis(&block));
- storage
+
+ let mut child_storage = ChildrenStorageOverlay::default();
+ child_storage.insert(
+ well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect(),
+ vec![(b"key".to_vec(), vec![42_u8])].into_iter().collect()
+ );
+
+ (storage, child_storage)
}
impl<Block: BlockT> client::light::fetcher::Fetcher<Block> for LightFetcher {
diff --git a/substrate/node-template/runtime/wasm/Cargo.lock b/substrate/node-template/runtime/wasm/Cargo.lock
index bd69f55c624..cfbf85fbf2d 100644
--- a/substrate/node-template/runtime/wasm/Cargo.lock
+++ b/substrate/node-template/runtime/wasm/Cargo.lock
@@ -2359,6 +2359,7 @@ version = "2.0.0"
dependencies = [
"parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sr-io 2.0.0",
"sr-primitives 2.0.0",
"sr-std 2.0.0",
"srml-support 2.0.0",
--
GitLab