From 3f30f69b5b20d99dc81611998659f14359976948 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 16 Jun 2020 13:14:49 +0200
Subject: [PATCH] historical slashing w ocw w adhoc tree creation (#6220)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* draft
* steps
* chore: fmt
* step by step
* more details
* make test public
* refactor: split into on and offchain
* test stab
* tabs my friend
* offchain overlay: split key into prefix and true key
Simplifies inspection and makes key actually unique.
* test: share state
* fix & test
* docs improv
* address review comments
* cleanup test chore
* refactor, abbrev link text
* chore: linewidth
* fix prefix key split fallout
* minor fallout
* minor changes
* addresses review comments
* rename historical.rs -> historical/mod.rs
* avoid shared::* wildcard import
* fix: add missing call to store_session_validator_set_to_offchain
* fix/compile: missing shared:: prefix
* fix/test: flow
* fix/review: Apply suggestions from code review
Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
* fix/review: more review comment fixes
* fix/review: make ValidatorSet private
* fix/include: core -> sp_core
* fix/review: fallout
* fix/visbility: make them public API
Ref #6358
* fix/review: review changes fallout - again
Co-authored-by: Bernhard Schuster <bernhard@parity.io>
Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
---
substrate/client/db/src/lib.rs | 7 +-
substrate/frame/session/Cargo.toml | 6 +-
.../src/{historical.rs => historical/mod.rs} | 17 +-
.../frame/session/src/historical/offchain.rs | 263 ++++++++++++++++++
.../frame/session/src/historical/onchain.rs | 62 +++++
.../frame/session/src/historical/shared.rs | 39 +++
.../primitives/core/src/offchain/storage.rs | 35 +--
.../primitives/core/src/offchain/testing.rs | 87 +++++-
.../primitives/state-machine/src/testing.rs | 19 +-
9 files changed, 496 insertions(+), 39 deletions(-)
rename substrate/frame/session/src/{historical.rs => historical/mod.rs} (97%)
create mode 100644 substrate/frame/session/src/historical/offchain.rs
create mode 100644 substrate/frame/session/src/historical/onchain.rs
create mode 100644 substrate/frame/session/src/historical/shared.rs
diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs
index f75693ec9f0..3bae2345675 100644
--- a/substrate/client/db/src/lib.rs
+++ b/substrate/client/db/src/lib.rs
@@ -544,7 +544,12 @@ pub struct BlockImportOperation<Block: BlockT> {
impl<Block: BlockT> BlockImportOperation<Block> {
fn apply_offchain(&mut self, transaction: &mut Transaction<DbHash>) {
- for (key, value_operation) in self.offchain_storage_updates.drain() {
+ for ((prefix, key), value_operation) in self.offchain_storage_updates.drain() {
+ let key: Vec<u8> = prefix
+ .into_iter()
+ .chain(sp_core::sp_std::iter::once(b'/'))
+ .chain(key.into_iter())
+ .collect();
match value_operation {
OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val),
OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key),
diff --git a/substrate/frame/session/Cargo.toml b/substrate/frame/session/Cargo.toml
index 6955940dc4d..6b74e3ef5f7 100644
--- a/substrate/frame/session/Cargo.toml
+++ b/substrate/frame/session/Cargo.toml
@@ -14,7 +14,9 @@ targets = ["x86_64-unknown-linux-gnu"]
[dependencies]
serde = { version = "1.0.101", optional = true }
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
+sp-core = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/core" }
sp-std = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/std" }
+sp-io = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/io" }
sp-runtime = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/runtime" }
sp-session = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/session" }
sp-staking = { version = "2.0.0-rc3", default-features = false, path = "../../primitives/staking" }
@@ -25,8 +27,6 @@ sp-trie = { version = "2.0.0-rc3", optional = true, default-features = false, pa
impl-trait-for-tuples = "0.1.3"
[dev-dependencies]
-sp-core = { version = "2.0.0-rc3", path = "../../primitives/core" }
-sp-io ={ version = "2.0.0-rc3", path = "../../primitives/io" }
sp-application-crypto = { version = "2.0.0-rc3", path = "../../primitives/application-crypto" }
lazy_static = "1.4.0"
@@ -37,7 +37,9 @@ std = [
"serde",
"codec/std",
"sp-std/std",
+ "sp-io/std",
"frame-support/std",
+ "sp-core/std",
"sp-runtime/std",
"sp-session/std",
"sp-staking/std",
diff --git a/substrate/frame/session/src/historical.rs b/substrate/frame/session/src/historical/mod.rs
similarity index 97%
rename from substrate/frame/session/src/historical.rs
rename to substrate/frame/session/src/historical/mod.rs
index a1c286eb392..20c3d57464c 100644
--- a/substrate/frame/session/src/historical.rs
+++ b/substrate/frame/session/src/historical/mod.rs
@@ -37,6 +37,10 @@ use sp_trie::{MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX};
use sp_trie::trie_types::{TrieDBMut, TrieDB};
use super::{SessionIndex, Module as SessionModule};
+mod shared;
+pub mod offchain;
+pub mod onchain;
+
/// Trait necessary for the historical module.
pub trait Trait: super::Trait {
/// Full identification of the validator.
@@ -116,6 +120,7 @@ impl<T: Trait, I> crate::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T
where I: SessionManager<T::ValidatorId, T::FullIdentification>
{
fn new_session(new_index: SessionIndex) -> Option<Vec<T::ValidatorId>> {
+
StoredRange::mutate(|range| {
range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1;
});
@@ -143,10 +148,13 @@ impl<T: Trait, I> crate::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T
new_validators
}
+
fn start_session(start_index: SessionIndex) {
<I as SessionManager<_, _>>::start_session(start_index)
}
+
fn end_session(end_index: SessionIndex) {
+ onchain::store_session_validator_set_to_offchain::<T>(end_index);
<I as SessionManager<_, _>>::end_session(end_index)
}
}
@@ -154,7 +162,7 @@ impl<T: Trait, I> crate::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T
/// A tuple of the validator's ID and their full identification.
pub type IdentificationTuple<T> = (<T as crate::Trait>::ValidatorId, <T as Trait>::FullIdentification);
-/// a trie instance for checking and generating proofs.
+/// A trie instance for checking and generating proofs.
pub struct ProvingTrie<T: Trait> {
db: MemoryDB<T::Hashing>,
root: T::Hash,
@@ -250,7 +258,6 @@ impl<T: Trait> ProvingTrie<T> {
.ok()?
.and_then(|raw| <IdentificationTuple<T>>::decode(&mut &*raw).ok())
}
-
}
impl<T: Trait, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)>
@@ -311,9 +318,9 @@ impl<T: Trait, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyTy
}
#[cfg(test)]
-mod tests {
+pub(crate) mod tests {
use super::*;
- use sp_core::crypto::key_types::DUMMY;
+ use sp_runtime::key_types::DUMMY;
use sp_runtime::testing::UintAuthorityId;
use crate::mock::{
NEXT_VALIDATORS, force_new_session,
@@ -323,7 +330,7 @@ mod tests {
type Historical = Module<Test>;
- fn new_test_ext() -> sp_io::TestExternalities {
+ pub(crate) fn new_test_ext() -> sp_io::TestExternalities {
let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
crate::GenesisConfig::<Test> {
keys: NEXT_VALIDATORS.with(|l|
diff --git a/substrate/frame/session/src/historical/offchain.rs b/substrate/frame/session/src/historical/offchain.rs
new file mode 100644
index 00000000000..97655d1a18b
--- /dev/null
+++ b/substrate/frame/session/src/historical/offchain.rs
@@ -0,0 +1,263 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Off-chain logic for creating a proof based data provided by on-chain logic.
+//!
+//! Validator-set extracting an iterator from an off-chain worker stored list containing
+//! historical validator-sets.
+//! Based on the logic of historical slashing, but the validation is done off-chain.
+//! Use [`fn store_current_session_validator_set_to_offchain()`](super::onchain) to store the
+//! required data to the offchain validator set.
+//! This is used in conjunction with [`ProvingTrie`](super::ProvingTrie) and
+//! the off-chain indexing API.
+
+use sp_runtime::{offchain::storage::StorageValueRef, KeyTypeId};
+use sp_session::MembershipProof;
+
+use super::super::{Module as SessionModule, SessionIndex};
+use super::{IdentificationTuple, ProvingTrie, Trait};
+
+use super::shared;
+use sp_std::prelude::*;
+
+
+/// A set of validators, which was used for a fixed session index.
+struct ValidatorSet<T: Trait> {
+ validator_set: Vec<IdentificationTuple<T>>,
+}
+
+impl<T: Trait> ValidatorSet<T> {
+ /// Load the set of validators for a particular session index from the off-chain storage.
+ ///
+ /// If none is found or decodable given `prefix` and `session`, it will return `None`.
+ /// Empty validator sets should only ever exist for genesis blocks.
+ pub fn load_from_offchain_db(session_index: SessionIndex) -> Option<Self> {
+ let derived_key = shared::derive_key(shared::PREFIX, session_index);
+ StorageValueRef::persistent(derived_key.as_ref())
+ .get::<Vec<(T::ValidatorId, T::FullIdentification)>>()
+ .flatten()
+ .map(|validator_set| Self { validator_set })
+ }
+
+ #[inline]
+ fn len(&self) -> usize {
+ self.validator_set.len()
+ }
+}
+
+/// Implement conversion into iterator for usage
+/// with [ProvingTrie](super::ProvingTrie::generate_for).
+impl<T: Trait> sp_std::iter::IntoIterator for ValidatorSet<T> {
+ type Item = (T::ValidatorId, T::FullIdentification);
+ type IntoIter = sp_std::vec::IntoIter<Self::Item>;
+ fn into_iter(self) -> Self::IntoIter {
+ self.validator_set.into_iter()
+ }
+}
+
+/// Create a proof based on the data available in the off-chain database.
+///
+/// Based on the yielded `MembershipProof` the implementer may decide what
+/// to do, i.e. in case of a failed proof, enqueue a transaction back on
+/// chain reflecting that, with all its consequences such as i.e. slashing.
+pub fn prove_session_membership<T: Trait, D: AsRef<[u8]>>(
+ session_index: SessionIndex,
+ session_key: (KeyTypeId, D),
+) -> Option<MembershipProof> {
+ let validators = ValidatorSet::<T>::load_from_offchain_db(session_index)?;
+ let count = validators.len() as u32;
+ let trie = ProvingTrie::<T>::generate_for(validators.into_iter()).ok()?;
+
+ let (id, data) = session_key;
+ trie.prove(id, data.as_ref())
+ .map(|trie_nodes| MembershipProof {
+ session: session_index,
+ trie_nodes,
+ validator_count: count,
+ })
+}
+
+
+/// Attempt to prune anything that is older than `first_to_keep` session index.
+///
+/// Due to re-organisation it could be that the `first_to_keep` might be less
+/// than the stored one, in which case the conservative choice is made to keep records
+/// up to the one that is the lesser.
+pub fn prune_older_than<T: Trait>(first_to_keep: SessionIndex) {
+ let derived_key = shared::LAST_PRUNE.to_vec();
+ let entry = StorageValueRef::persistent(derived_key.as_ref());
+ match entry.mutate(|current: Option<Option<SessionIndex>>| -> Result<_, ()> {
+ match current {
+ Some(Some(current)) if current < first_to_keep => Ok(first_to_keep),
+ // do not move the cursor, if the new one would be behind ours
+ Some(Some(current)) => Ok(current),
+ None => Ok(first_to_keep),
+ // if the storage contains undecodable data, overwrite with current anyways
+ // which might leak some entries being never purged, but that is acceptable
+ // in this context
+ Some(None) => Ok(first_to_keep),
+ }
+ }) {
+ Ok(Ok(new_value)) => {
+ // on a re-org this is not necessarily true, with the above they might be equal
+ if new_value < first_to_keep {
+ for session_index in new_value..first_to_keep {
+ let derived_key = shared::derive_key(shared::PREFIX, session_index);
+ let _ = StorageValueRef::persistent(derived_key.as_ref()).clear();
+ }
+ }
+ }
+ Ok(Err(_)) => {} // failed to store the value calculated with the given closure
+ Err(_) => {} // failed to calculate the value to store with the given closure
+ }
+}
+
+/// Keep the newest `n` items, and prune all items older than that.
+pub fn keep_newest<T: Trait>(n_to_keep: usize) {
+ let session_index = <SessionModule<T>>::current_index();
+ let n_to_keep = n_to_keep as SessionIndex;
+ if n_to_keep < session_index {
+ prune_older_than::<T>(session_index - n_to_keep)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{onchain, Module};
+ use super::*;
+ use crate::mock::{
+ force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS,
+ };
+ use codec::Encode;
+ use frame_support::traits::{KeyOwnerProofSystem, OnInitialize};
+ use sp_core::crypto::key_types::DUMMY;
+ use sp_core::offchain::{
+ testing::TestOffchainExt,
+ OffchainExt,
+ StorageKind,
+ };
+
+ use sp_runtime::testing::UintAuthorityId;
+
+ type Historical = Module<Test>;
+
+ pub fn new_test_ext() -> sp_io::TestExternalities {
+ let mut ext = frame_system::GenesisConfig::default()
+ .build_storage::<Test>()
+ .expect("Failed to create test externalities.");
+
+ crate::GenesisConfig::<Test> {
+ keys: NEXT_VALIDATORS.with(|l| {
+ l.borrow()
+ .iter()
+ .cloned()
+ .map(|i| (i, i, UintAuthorityId(i).into()))
+ .collect()
+ }),
+ }
+ .assimilate_storage(&mut ext)
+ .unwrap();
+
+
+ let mut ext = sp_io::TestExternalities::new(ext);
+
+ let (offchain, offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+
+ const ITERATIONS: u32 = 5u32;
+ let mut seed = [0u8; 32];
+ seed[0..4].copy_from_slice(&ITERATIONS.to_le_bytes());
+ offchain_state.write().seed = seed;
+
+ ext.register_extension(OffchainExt::new(offchain));
+ ext
+ }
+
+ #[test]
+ fn encode_decode_roundtrip() {
+ use codec::{Decode, Encode};
+ use super::super::super::Trait as SessionTrait;
+ use super::super::Trait as HistoricalTrait;
+
+ let sample = (
+ 22u32 as <Test as SessionTrait>::ValidatorId,
+ 7_777_777 as <Test as HistoricalTrait>::FullIdentification);
+
+ let encoded = sample.encode();
+ let decoded = Decode::decode(&mut encoded.as_slice()).expect("Must decode");
+ assert_eq!(sample, decoded);
+ }
+
+ #[test]
+ fn onchain_to_offchain() {
+ let mut ext = new_test_ext();
+
+ const DATA: &[u8] = &[7,8,9,10,11];
+ ext.execute_with(|| {
+ b"alphaomega"[..].using_encoded(|key| sp_io::offchain_index::set(key, DATA));
+ });
+
+ ext.persist_offchain_overlay();
+
+ ext.execute_with(|| {
+ let data =
+ b"alphaomega"[..].using_encoded(|key| {
+ sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, key)
+ });
+ assert_eq!(data, Some(DATA.to_vec()));
+ });
+ }
+
+
+ #[test]
+ fn historical_proof_offchain() {
+ let mut ext = new_test_ext();
+ let encoded_key_1 = UintAuthorityId(1).encode();
+
+ ext.execute_with(|| {
+ set_next_validators(vec![1, 2]);
+ force_new_session();
+
+ System::set_block_number(1);
+ Session::on_initialize(1);
+
+ // "on-chain"
+ onchain::store_current_session_validator_set_to_offchain::<Test>();
+ assert_eq!(<SessionModule<Test>>::current_index(), 1);
+
+ set_next_validators(vec![7, 8]);
+
+ force_new_session();
+ });
+
+ ext.persist_offchain_overlay();
+
+ ext.execute_with(|| {
+
+
+ System::set_block_number(2);
+ Session::on_initialize(2);
+ assert_eq!(<SessionModule<Test>>::current_index(), 2);
+
+ // "off-chain"
+ let proof = prove_session_membership::<Test, _>(1, (DUMMY, &encoded_key_1));
+ assert!(proof.is_some());
+ let proof = proof.expect("Must be Some(Proof)");
+
+ assert!(Historical::check_proof((DUMMY, &encoded_key_1[..]), proof.clone()).is_some());
+ });
+ }
+}
diff --git a/substrate/frame/session/src/historical/onchain.rs b/substrate/frame/session/src/historical/onchain.rs
new file mode 100644
index 00000000000..745603a4982
--- /dev/null
+++ b/substrate/frame/session/src/historical/onchain.rs
@@ -0,0 +1,62 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! On-chain logic to store a validator-set for deferred validation using an off-chain worker.
+
+use codec::Encode;
+use sp_runtime::traits::Convert;
+
+use super::super::Trait as SessionTrait;
+use super::super::{Module as SessionModule, SessionIndex};
+use super::Trait as HistoricalTrait;
+
+use super::shared;
+use sp_std::prelude::*;
+
+/// Store the validator-set associated to the `session_index` to the off-chain database.
+///
+/// Further processing is then done [`off-chain side`](super::offchain).
+///
+/// **Must** be called from on-chain, i.e. a call that originates from
+/// `on_initialize(..)` or `on_finalization(..)`.
+/// **Must** be called during the session, which validator-set is to be stored for further
+/// off-chain processing. Otherwise the `FullIdentification` might not be available.
+pub fn store_session_validator_set_to_offchain<T: HistoricalTrait + SessionTrait>(
+ session_index: SessionIndex,
+) {
+ let encoded_validator_list = <SessionModule<T>>::validators()
+ .into_iter()
+ .filter_map(|validator_id: <T as SessionTrait>::ValidatorId| {
+ let full_identification =
+ <<T as HistoricalTrait>::FullIdentificationOf>::convert(validator_id.clone());
+ full_identification.map(|full_identification| (validator_id, full_identification))
+ })
+ .collect::<Vec<_>>();
+
+ encoded_validator_list.using_encoded(|encoded_validator_list| {
+ let derived_key = shared::derive_key(shared::PREFIX, session_index);
+ sp_io::offchain_index::set(derived_key.as_slice(), encoded_validator_list);
+ });
+}
+
+/// Store the validator set associated to the _current_ session index to the off-chain database.
+///
+/// See [`fn store_session_validator_set_...(..)`](Self::store_session_validator_set_to_offchain)
+/// for further information and restrictions.
+pub fn store_current_session_validator_set_to_offchain<T: HistoricalTrait + SessionTrait>() {
+ store_session_validator_set_to_offchain::<T>(<SessionModule<T>>::current_index());
+}
diff --git a/substrate/frame/session/src/historical/shared.rs b/substrate/frame/session/src/historical/shared.rs
new file mode 100644
index 00000000000..fda0361b059
--- /dev/null
+++ b/substrate/frame/session/src/historical/shared.rs
@@ -0,0 +1,39 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared logic between on-chain and off-chain components used for slashing using an off-chain
+//! worker.
+
+
+use super::SessionIndex;
+use sp_std::prelude::*;
+use codec::Encode;
+
+pub(super) const PREFIX: &[u8] = b"session_historical";
+pub(super) const LAST_PRUNE: &[u8] = b"session_historical_last_prune";
+
+/// Derive the key used to store the list of validators
+pub(super) fn derive_key<P: AsRef<[u8]>>(prefix: P, session_index: SessionIndex) -> Vec<u8> {
+ let prefix: &[u8] = prefix.as_ref();
+ session_index.using_encoded(|encoded_session_index| {
+ prefix.into_iter()
+ .chain(b"/".into_iter())
+ .chain(encoded_session_index.into_iter())
+ .copied()
+ .collect::<Vec<u8>>()
+ })
+}
\ No newline at end of file
diff --git a/substrate/primitives/core/src/offchain/storage.rs b/substrate/primitives/core/src/offchain/storage.rs
index 52a7bbe857d..7d7c711ed95 100644
--- a/substrate/primitives/core/src/offchain/storage.rs
+++ b/substrate/primitives/core/src/offchain/storage.rs
@@ -101,8 +101,9 @@ pub enum OffchainOverlayedChange {
pub enum OffchainOverlayedChanges {
/// Writing overlay changes to the offchain worker database is disabled by configuration.
Disabled,
- /// Overlay changes can be recorded using the inner collection of this variant.
- Enabled(HashMap<Vec<u8>, OffchainOverlayedChange>),
+ /// Overlay changes can be recorded using the inner collection of this variant,
+ /// where the identifier is the tuple of `(prefix, key)`.
+ Enabled(HashMap<(Vec<u8>, Vec<u8>), OffchainOverlayedChange>),
}
impl Default for OffchainOverlayedChanges {
@@ -140,23 +141,21 @@ impl OffchainOverlayedChanges {
/// Remove a key and its associated value from the offchain database.
pub fn remove(&mut self, prefix: &[u8], key: &[u8]) {
if let Self::Enabled(ref mut storage) = self {
- let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
- let _ = storage.insert(key, OffchainOverlayedChange::Remove);
+ let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove);
}
}
/// Set the value associated with a key under a prefix to the value provided.
pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
if let Self::Enabled(ref mut storage) = self {
- let key = prefix.iter().chain(key).cloned().collect();
- let _ = storage.insert(key, OffchainOverlayedChange::SetValue(value.to_vec()));
+ let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::SetValue(value.to_vec()));
}
}
/// Obtain a associated value to the given key in storage with prefix.
pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option<OffchainOverlayedChange> {
if let Self::Enabled(ref storage) = self {
- let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+ let key = (prefix.to_vec(), key.to_vec());
storage.get(&key).cloned()
} else {
None
@@ -168,11 +167,11 @@ use std::collections::hash_map;
/// Iterate by reference over the prepared offchain worker storage changes.
pub struct OffchainOverlayedChangesIter<'i> {
- inner: Option<hash_map::Iter<'i, Vec<u8>, OffchainOverlayedChange>>,
+ inner: Option<hash_map::Iter<'i, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
}
impl<'i> Iterator for OffchainOverlayedChangesIter<'i> {
- type Item = (&'i Vec<u8>, &'i OffchainOverlayedChange);
+ type Item = (&'i (Vec<u8>, Vec<u8>), &'i OffchainOverlayedChange);
fn next(&mut self) -> Option<Self::Item> {
if let Some(ref mut iter) = self.inner {
iter.next()
@@ -197,11 +196,11 @@ impl<'i> OffchainOverlayedChangesIter<'i> {
/// Iterate by value over the prepared offchain worker storage changes.
pub struct OffchainOverlayedChangesIntoIter {
- inner: Option<hash_map::IntoIter<Vec<u8>,OffchainOverlayedChange>>,
+ inner: Option<hash_map::IntoIter<(Vec<u8>,Vec<u8>),OffchainOverlayedChange>>,
}
impl Iterator for OffchainOverlayedChangesIntoIter {
- type Item = (Vec<u8>, OffchainOverlayedChange);
+ type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
fn next(&mut self) -> Option<Self::Item> {
if let Some(ref mut iter) = self.inner {
iter.next()
@@ -225,11 +224,11 @@ impl OffchainOverlayedChangesIntoIter {
/// Iterate over all items while draining them from the collection.
pub struct OffchainOverlayedChangesDrain<'d> {
- inner: Option<hash_map::Drain<'d, Vec<u8>,OffchainOverlayedChange>>,
+ inner: Option<hash_map::Drain<'d, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
}
impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> {
- type Item = (Vec<u8>, OffchainOverlayedChange);
+ type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
fn next(&mut self) -> Option<Self::Item> {
if let Some(ref mut iter) = self.inner {
iter.next()
@@ -286,9 +285,13 @@ mod test {
ooc.set(STORAGE_PREFIX, b"ppp", b"rrr");
let mut iter = ooc.into_iter();
- let mut k = STORAGE_PREFIX.to_vec();
- k.extend_from_slice(&b"ppp"[..]);
- assert_eq!(iter.next(), Some((k, OffchainOverlayedChange::SetValue(b"rrr".to_vec()))));
+ assert_eq!(
+ iter.next(),
+ Some(
+ ((STORAGE_PREFIX.to_vec(), b"ppp".to_vec()),
+ OffchainOverlayedChange::SetValue(b"rrr".to_vec()))
+ )
+ );
assert_eq!(iter.next(), None);
}
}
diff --git a/substrate/primitives/core/src/offchain/testing.rs b/substrate/primitives/core/src/offchain/testing.rs
index 76cf8915f20..a14e906f543 100644
--- a/substrate/primitives/core/src/offchain/testing.rs
+++ b/substrate/primitives/core/src/offchain/testing.rs
@@ -26,7 +26,7 @@ use std::{
};
use crate::offchain::{
self,
- storage::InMemOffchainStorage,
+ storage::{InMemOffchainStorage, OffchainOverlayedChange, OffchainOverlayedChanges},
HttpError,
HttpRequestId as RequestId,
HttpRequestStatus as RequestStatus,
@@ -36,6 +36,7 @@ use crate::offchain::{
TransactionPool,
OffchainStorage,
};
+
use parking_lot::RwLock;
/// Pending request.
@@ -61,6 +62,57 @@ pub struct PendingRequest {
pub response_headers: Vec<(String, String)>,
}
+/// Sharable "persistent" offchain storage for test.
+#[derive(Debug, Clone, Default)]
+pub struct TestPersistentOffchainDB {
+ persistent: Arc<RwLock<InMemOffchainStorage>>,
+}
+
+impl TestPersistentOffchainDB {
+ /// Create a new and empty offchain storage db for persistent items
+ pub fn new() -> Self {
+ Self {
+ persistent: Arc::new(RwLock::new(InMemOffchainStorage::default()))
+ }
+ }
+
+ /// Apply a set of off-chain changes directly to the test backend
+ pub fn apply_offchain_changes(&mut self, changes: &mut OffchainOverlayedChanges) {
+ let mut me = self.persistent.write();
+ for ((_prefix, key), value_operation) in changes.drain() {
+ match value_operation {
+ OffchainOverlayedChange::SetValue(val) => me.set(b"", key.as_slice(), val.as_slice()),
+ OffchainOverlayedChange::Remove => me.remove(b"", key.as_slice()),
+ }
+ }
+ }
+}
+
+impl OffchainStorage for TestPersistentOffchainDB {
+ fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+ self.persistent.write().set(prefix, key, value);
+ }
+
+ fn remove(&mut self, prefix: &[u8], key: &[u8]) {
+ self.persistent.write().remove(prefix, key);
+ }
+
+ fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+ self.persistent.read().get(prefix, key)
+ }
+
+ fn compare_and_set(
+ &mut self,
+ prefix: &[u8],
+ key: &[u8],
+ old_value: Option<&[u8]>,
+ new_value: &[u8],
+ ) -> bool {
+ self.persistent.write().compare_and_set(prefix, key, old_value, new_value)
+ }
+}
+
+
/// Internal state of the externalities.
///
/// This can be used in tests to respond or assert stuff about interactions.
@@ -70,7 +122,7 @@ pub struct OffchainState {
pub requests: BTreeMap<RequestId, PendingRequest>,
expected_requests: BTreeMap<RequestId, PendingRequest>,
/// Persistent local storage
- pub persistent_storage: InMemOffchainStorage,
+ pub persistent_storage: TestPersistentOffchainDB,
/// Local storage
pub local_storage: InMemOffchainStorage,
/// A supposedly random seed.
@@ -145,6 +197,13 @@ impl TestOffchainExt {
let state = ext.0.clone();
(ext, state)
}
+
+ /// Create new `TestOffchainExt` and a reference to the internal state.
+ pub fn with_offchain_db(offchain_db: TestPersistentOffchainDB) -> (Self, Arc<RwLock<OffchainState>>) {
+ let (ext, state) = Self::new();
+ ext.0.write().persistent_storage = offchain_db;
+ (ext, state)
+ }
}
impl offchain::Externalities for TestOffchainExt {
@@ -174,17 +233,17 @@ impl offchain::Externalities for TestOffchainExt {
fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
let mut state = self.0.write();
match kind {
- StorageKind::LOCAL => &mut state.local_storage,
- StorageKind::PERSISTENT => &mut state.persistent_storage,
- }.set(b"", key, value);
+ StorageKind::LOCAL => state.local_storage.set(b"", key, value),
+ StorageKind::PERSISTENT => state.persistent_storage.set(b"", key, value),
+ };
}
fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) {
let mut state = self.0.write();
match kind {
- StorageKind::LOCAL => &mut state.local_storage,
- StorageKind::PERSISTENT => &mut state.persistent_storage,
- }.remove(b"", key);
+ StorageKind::LOCAL => state.local_storage.remove(b"", key),
+ StorageKind::PERSISTENT => state.persistent_storage.remove(b"", key),
+ };
}
fn local_storage_compare_and_set(
@@ -196,17 +255,17 @@ impl offchain::Externalities for TestOffchainExt {
) -> bool {
let mut state = self.0.write();
match kind {
- StorageKind::LOCAL => &mut state.local_storage,
- StorageKind::PERSISTENT => &mut state.persistent_storage,
- }.compare_and_set(b"", key, old_value, new_value)
+ StorageKind::LOCAL => state.local_storage.compare_and_set(b"", key, old_value, new_value),
+ StorageKind::PERSISTENT => state.persistent_storage.compare_and_set(b"", key, old_value, new_value),
+ }
}
fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
let state = self.0.read();
match kind {
- StorageKind::LOCAL => &state.local_storage,
- StorageKind::PERSISTENT => &state.persistent_storage,
- }.get(b"", key)
+ StorageKind::LOCAL => state.local_storage.get(b"", key),
+ StorageKind::PERSISTENT => state.persistent_storage.get(b"", key),
+ }
}
fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<RequestId, ()> {
diff --git a/substrate/primitives/state-machine/src/testing.rs b/substrate/primitives/state-machine/src/testing.rs
index 71124a68bb5..2ea2961830f 100644
--- a/substrate/primitives/state-machine/src/testing.rs
+++ b/substrate/primitives/state-machine/src/testing.rs
@@ -31,7 +31,10 @@ use crate::{
},
};
use sp_core::{
- offchain::storage::OffchainOverlayedChanges,
+ offchain::{
+ testing::TestPersistentOffchainDB,
+ storage::OffchainOverlayedChanges
+ },
storage::{
well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
Storage,
@@ -47,6 +50,7 @@ where
{
overlay: OverlayedChanges,
offchain_overlay: OffchainOverlayedChanges,
+ offchain_db: TestPersistentOffchainDB,
storage_transaction_cache: StorageTransactionCache<
<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
>,
@@ -108,9 +112,12 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor()));
+ let offchain_db = TestPersistentOffchainDB::new();
+
TestExternalities {
overlay,
offchain_overlay,
+ offchain_db,
changes_trie_config,
extensions,
changes_trie_storage: ChangesTrieInMemoryStorage::new(),
@@ -119,6 +126,16 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
}
}
+ /// Move offchain changes from overlay to the persistent store.
+ pub fn persist_offchain_overlay(&mut self) {
+ self.offchain_db.apply_offchain_changes(&mut self.offchain_overlay);
+ }
+
+ /// A shared reference type around the offchain worker storage.
+ pub fn offchain_db(&self) -> TestPersistentOffchainDB {
+ self.offchain_db.clone()
+ }
+
/// Insert key/value into backend
pub fn insert(&mut self, k: StorageKey, v: StorageValue) {
self.backend.insert(vec![(None, vec![(k, Some(v))])]);
--
GitLab