diff --git a/substrate/frame/session/benchmarking/src/lib.rs b/substrate/frame/session/benchmarking/src/lib.rs index 8ca713b1bbf61ed0ae94f0cdca96e5b7737d6c36..6d9d81f38517686d2720279fa5698e6ab8636bc8 100644 --- a/substrate/frame/session/benchmarking/src/lib.rs +++ b/substrate/frame/session/benchmarking/src/lib.rs @@ -22,6 +22,7 @@ mod mock; +use sp_runtime::traits::{One, StaticLookup}; use sp_std::{prelude::*, vec}; use frame_benchmarking::benchmarks; @@ -30,12 +31,11 @@ use frame_support::{ traits::{KeyOwnerProofSystem, OnInitialize}, }; use frame_system::RawOrigin; -use pallet_session::{historical::Module as Historical, Pallet as Session, *}; +use pallet_session::{historical::Pallet as Historical, Pallet as Session, *}; use pallet_staking::{ benchmarking::create_validator_with_nominators, testing_utils::create_validators, RewardDestination, }; -use sp_runtime::traits::{One, StaticLookup}; const MAX_VALIDATORS: u32 = 1000; diff --git a/substrate/frame/session/src/historical/mod.rs b/substrate/frame/session/src/historical/mod.rs index 0801b2aca1701b04af2418c6b390eb6bde6ce551..a3e64f4f9efa41b6ce149e2b6494066e64cec86b 100644 --- a/substrate/frame/session/src/historical/mod.rs +++ b/substrate/frame/session/src/historical/mod.rs @@ -26,62 +26,74 @@ //! These roots and proofs of inclusion can be generated at any time during the current session. //! Afterwards, the proofs can be fed to a consensus module when reporting misbehavior. -use super::{Pallet as SessionModule, SessionIndex}; +pub mod offchain; +pub mod onchain; +mod shared; + use codec::{Decode, Encode}; -use frame_support::{ - decl_module, decl_storage, print, - traits::{ValidatorSet, ValidatorSetWithIdentification}, - Parameter, -}; use sp_runtime::{ traits::{Convert, OpaqueKeys}, KeyTypeId, }; use sp_session::{MembershipProof, ValidatorCount}; +use sp_staking::SessionIndex; use sp_std::prelude::*; use sp_trie::{ trie_types::{TrieDB, TrieDBMut}, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -pub mod offchain; -pub mod onchain; -mod shared; +use frame_support::{ + print, + traits::{KeyOwnerProofSystem, StorageVersion, ValidatorSet, ValidatorSetWithIdentification}, + Parameter, +}; -/// Config necessary for the historical module. -pub trait Config: super::Config { - /// Full identification of the validator. - type FullIdentification: Parameter; - - /// A conversion from validator ID to full identification. - /// - /// This should contain any references to economic actors associated with the - /// validator, since they may be outdated by the time this is queried from a - /// historical trie. - /// - /// It must return the identification for the current session index. - type FullIdentificationOf: Convert<Self::ValidatorId, Option<Self::FullIdentification>>; -} +use crate::{self as pallet_session, Pallet as Session}; + +pub use pallet::*; -decl_storage! { - trait Store for Module<T: Config> as Session { - /// Mapping from historical session indices to session-data root hash and validator count. - HistoricalSessions get(fn historical_root): - map hasher(twox_64_concat) SessionIndex => Option<(T::Hash, ValidatorCount)>; - /// The range of historical sessions we store. [first, last) - StoredRange: Option<(SessionIndex, SessionIndex)>; - /// Deprecated. - CachedObsolete: - map hasher(twox_64_concat) SessionIndex - => Option<Vec<(T::ValidatorId, T::FullIdentification)>>; +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::storage_version(STORAGE_VERSION)] + pub struct Pallet<T>(_); + + /// Config necessary for the historical pallet. + #[pallet::config] + pub trait Config: pallet_session::Config + frame_system::Config { + /// Full identification of the validator. + type FullIdentification: Parameter; + + /// A conversion from validator ID to full identification. + /// + /// This should contain any references to economic actors associated with the + /// validator, since they may be outdated by the time this is queried from a + /// historical trie. + /// + /// It must return the identification for the current session index. + type FullIdentificationOf: Convert<Self::ValidatorId, Option<Self::FullIdentification>>; } -} -decl_module! { - pub struct Module<T: Config> for enum Call where origin: T::Origin {} + /// Mapping from historical session indices to session-data root hash and validator count. + #[pallet::storage] + #[pallet::getter(fn historical_root)] + pub type HistoricalSessions<T: Config> = + StorageMap<_, Twox64Concat, SessionIndex, (T::Hash, ValidatorCount), OptionQuery>; + + /// The range of historical sessions we store. [first, last) + #[pallet::storage] + pub type StoredRange<T> = StorageValue<_, (SessionIndex, SessionIndex), OptionQuery>; } -impl<T: Config> Module<T> { +impl<T: Config> Pallet<T> { /// Prune historical stored session roots up to (but not including) /// `up_to`. pub fn prune_up_to(up_to: SessionIndex) { @@ -109,7 +121,7 @@ impl<T: Config> Module<T> { } } -impl<T: Config> ValidatorSet<T::AccountId> for Module<T> { +impl<T: Config> ValidatorSet<T::AccountId> for Pallet<T> { type ValidatorId = T::ValidatorId; type ValidatorIdOf = T::ValidatorIdOf; @@ -122,7 +134,7 @@ impl<T: Config> ValidatorSet<T::AccountId> for Module<T> { } } -impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Module<T> { +impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Pallet<T> { type Identification = T::FullIdentification; type IdentificationOf = T::FullIdentificationOf; } @@ -130,7 +142,7 @@ impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Module<T> { /// Specialization of the crate-level `SessionManager` which returns the set of full identification /// when creating a new session. pub trait SessionManager<ValidatorId, FullIdentification>: - crate::SessionManager<ValidatorId> + pallet_session::SessionManager<ValidatorId> { /// If there was a validator set change, its returns the set of new validators along with their /// full identifications. @@ -150,7 +162,7 @@ pub struct NoteHistoricalRoot<T, I>(sp_std::marker::PhantomData<(T, I)>); impl<T: Config, I: SessionManager<T::ValidatorId, T::FullIdentification>> NoteHistoricalRoot<T, I> { fn do_new_session(new_index: SessionIndex, is_genesis: bool) -> Option<Vec<T::ValidatorId>> { - StoredRange::mutate(|range| { + <StoredRange<T>>::mutate(|range| { range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1; }); @@ -183,7 +195,7 @@ impl<T: Config, I: SessionManager<T::ValidatorId, T::FullIdentification>> NoteHi } } -impl<T: Config, I> crate::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T, I> +impl<T: Config, I> pallet_session::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T, I> where I: SessionManager<T::ValidatorId, T::FullIdentification>, { @@ -207,7 +219,7 @@ where /// A tuple of the validator's ID and their full identification. pub type IdentificationTuple<T> = - (<T as crate::Config>::ValidatorId, <T as Config>::FullIdentification); + (<T as pallet_session::Config>::ValidatorId, <T as Config>::FullIdentification); /// A trie instance for checking and generating proofs. pub struct ProvingTrie<T: Config> { @@ -227,7 +239,7 @@ impl<T: Config> ProvingTrie<T> { let mut trie = TrieDBMut::new(&mut db, &mut root); for (i, (validator, full_id)) in validators.into_iter().enumerate() { let i = i as u32; - let keys = match <SessionModule<T>>::load_keys(&validator) { + let keys = match <Session<T>>::load_keys(&validator) { None => continue, Some(k) => k, }; @@ -304,15 +316,13 @@ impl<T: Config> ProvingTrie<T> { } } -impl<T: Config, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)> - for Module<T> -{ +impl<T: Config, D: AsRef<[u8]>> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet<T> { type Proof = MembershipProof; type IdentificationTuple = IdentificationTuple<T>; fn prove(key: (KeyTypeId, D)) -> Option<Self::Proof> { - let session = <SessionModule<T>>::current_index(); - let validators = <SessionModule<T>>::validators() + let session = <Session<T>>::current_index(); + let validators = <Session<T>>::validators() .into_iter() .filter_map(|validator| { T::FullIdentificationOf::convert(validator.clone()) @@ -335,10 +345,10 @@ impl<T: Config, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyT fn check_proof(key: (KeyTypeId, D), proof: Self::Proof) -> Option<IdentificationTuple<T>> { let (id, data) = key; - if proof.session == <SessionModule<T>>::current_index() { - <SessionModule<T>>::key_owner(id, data.as_ref()).and_then(|owner| { + if proof.session == <Session<T>>::current_index() { + <Session<T>>::key_owner(id, data.as_ref()).and_then(|owner| { T::FullIdentificationOf::convert(owner.clone()).and_then(move |id| { - let count = <SessionModule<T>>::validators().len() as ValidatorCount; + let count = <Session<T>>::validators().len() as ValidatorCount; if count != proof.validator_count { return None @@ -374,7 +384,7 @@ pub(crate) mod tests { BasicExternalities, }; - type Historical = Module<Test>; + type Historical = Pallet<Test>; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap(); @@ -386,7 +396,9 @@ pub(crate) mod tests { frame_system::Pallet::<Test>::inc_providers(k); } }); - crate::GenesisConfig::<Test> { keys }.assimilate_storage(&mut t).unwrap(); + pallet_session::GenesisConfig::<Test> { keys } + .assimilate_storage(&mut t) + .unwrap(); sp_io::TestExternalities::new(t) } @@ -436,27 +448,27 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((0, 100))); + assert_eq!(<StoredRange<Test>>::get(), Some((0, 100))); for i in 0..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(10); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(<StoredRange<Test>>::get(), Some((10, 100))); Historical::prune_up_to(9); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(<StoredRange<Test>>::get(), Some((10, 100))); for i in 10..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(99); - assert_eq!(StoredRange::get(), Some((99, 100))); + assert_eq!(<StoredRange<Test>>::get(), Some((99, 100))); Historical::prune_up_to(100); - assert_eq!(StoredRange::get(), None); + assert_eq!(<StoredRange<Test>>::get(), None); for i in 99..199u64 { set_next_validators(vec![i]); @@ -466,14 +478,14 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((100, 200))); + assert_eq!(<StoredRange<Test>>::get(), Some((100, 200))); for i in 100..200 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(9999); - assert_eq!(StoredRange::get(), None); + assert_eq!(<StoredRange<Test>>::get(), None); for i in 100..200 { assert!(Historical::historical_root(i).is_none()) diff --git a/substrate/frame/session/src/historical/offchain.rs b/substrate/frame/session/src/historical/offchain.rs index b646ecc2764f72f4b75745ac2b4903109567b362..0b292b57658d01ad5d7eb527a2526f3f34774191 100644 --- a/substrate/frame/session/src/historical/offchain.rs +++ b/substrate/frame/session/src/historical/offchain.rs @@ -140,7 +140,7 @@ pub fn keep_newest<T: Config>(n_to_keep: usize) { mod tests { use super::*; use crate::{ - historical::{onchain, Module}, + historical::{onchain, Pallet}, mock::{force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS}, }; @@ -156,7 +156,7 @@ mod tests { BasicExternalities, }; - type Historical = Module<Test>; + type Historical = Pallet<Test>; pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default() diff --git a/substrate/frame/session/src/lib.rs b/substrate/frame/session/src/lib.rs index 6779285ee31878b8c635a2143e00d7f76925edbd..2fd34365705bb5742bd54e892fcaafefc368be57 100644 --- a/substrate/frame/session/src/lib.rs +++ b/substrate/frame/session/src/lib.rs @@ -108,6 +108,7 @@ #[cfg(feature = "historical")] pub mod historical; +pub mod migrations; #[cfg(test)] mod mock; #[cfg(test)] diff --git a/substrate/frame/session/src/migrations/mod.rs b/substrate/frame/session/src/migrations/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..ccc5ee3c2e5255cb373fd1b7b3c438397c581347 --- /dev/null +++ b/substrate/frame/session/src/migrations/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Version 1. +/// +/// In version 0 session historical pallet uses `Session` for storage module prefix. +/// In version 1 it uses its name as configured in `construct_runtime`. +/// This migration moves session historical pallet storages from old prefix to new prefix. +#[cfg(feature = "historical")] +pub mod v1; diff --git a/substrate/frame/session/src/migrations/v1.rs b/substrate/frame/session/src/migrations/v1.rs new file mode 100644 index 0000000000000000000000000000000000000000..1de199fe7bedddee913209517780115bd3af2c8a --- /dev/null +++ b/substrate/frame/session/src/migrations/v1.rs @@ -0,0 +1,194 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sp_io::hashing::twox_128; +use sp_std::str; + +use frame_support::{ + storage::{generator::StorageValue, StoragePrefixedMap}, + traits::{ + Get, GetStorageVersion, PalletInfoAccess, StorageVersion, + STORAGE_VERSION_STORAGE_KEY_POSTFIX, + }, + weights::Weight, +}; + +use crate::historical as pallet_session_historical; + +const OLD_PREFIX: &str = "Session"; + +/// Migrate the entire storage of this pallet to a new prefix. +/// +/// This new prefix must be the same as the one set in construct_runtime. +/// +/// The migration will look into the storage version in order not to trigger a migration on an up +/// to date storage. Thus the on chain storage version must be less than 1 in order to trigger the +/// migration. +pub fn migrate<T: pallet_session_historical::Config, P: GetStorageVersion + PalletInfoAccess>( +) -> Weight { + let new_pallet_name = <P as PalletInfoAccess>::name(); + + if new_pallet_name == OLD_PREFIX { + log::info!( + target: "runtime::session_historical", + "New pallet name is equal to the old prefix. No migration needs to be done.", + ); + return 0 + } + + let on_chain_storage_version = <P as GetStorageVersion>::on_chain_storage_version(); + log::info!( + target: "runtime::session_historical", + "Running migration to v1 for session_historical with storage version {:?}", + on_chain_storage_version, + ); + + if on_chain_storage_version < 1 { + let storage_prefix = pallet_session_historical::HistoricalSessions::<T>::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + let storage_prefix = pallet_session_historical::StoredRange::<T>::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + StorageVersion::new(1).put::<P>(); + <T as frame_system::Config>::BlockWeights::get().max_block + } else { + log::warn!( + target: "runtime::session_historical", + "Attempted to apply migration to v1 but failed because storage version is {:?}", + on_chain_storage_version, + ); + 0 + } +} + +/// Some checks prior to migration. This can be linked to +/// [`frame_support::traits::OnRuntimeUpgrade::pre_upgrade`] for further testing. +/// +/// Panics if anything goes wrong. +pub fn pre_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name = <P as PalletInfoAccess>::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::<T>::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::<T>::storage_prefix(); + + log_migration("pre-migration", storage_prefix_historical_sessions, OLD_PREFIX, new_pallet_name); + log_migration("pre-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let storage_version_key = twox_128(STORAGE_VERSION_STORAGE_KEY_POSTFIX); + + let mut new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |key| Ok(key.to_vec()), + ); + + // Ensure nothing except the storage_version_key is stored in the new prefix. + assert!(new_pallet_prefix_iter.all(|key| key == storage_version_key)); + + assert!(<P as GetStorageVersion>::on_chain_storage_version() < 1); +} + +/// Some checks for after migration. This can be linked to +/// [`frame_support::traits::OnRuntimeUpgrade::post_upgrade`] for further testing. +/// +/// Panics if anything goes wrong. +pub fn post_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name = <P as PalletInfoAccess>::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::<T>::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::<T>::storage_prefix(); + + log_migration( + "post-migration", + storage_prefix_historical_sessions, + OLD_PREFIX, + new_pallet_name, + ); + log_migration("post-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + // Assert that no `HistoricalSessions` and `StoredRange` storages remains at the old prefix. + let old_pallet_prefix = twox_128(OLD_PREFIX.as_bytes()); + let old_historical_sessions_key = + [&old_pallet_prefix, &twox_128(storage_prefix_historical_sessions)[..]].concat(); + let old_historical_sessions_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_historical_sessions_key.to_vec(), + old_historical_sessions_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_historical_sessions_key_iter.count(), 0); + + let old_stored_range_key = + [&old_pallet_prefix, &twox_128(storage_prefix_stored_range)[..]].concat(); + let old_stored_range_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_stored_range_key.to_vec(), + old_stored_range_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_stored_range_key_iter.count(), 0); + + // Assert that the `HistoricalSessions` and `StoredRange` storages (if they exist) have been + // moved to the new prefix. + // NOTE: storage_version_key is already in the new prefix. + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |_| Ok(()), + ); + assert!(new_pallet_prefix_iter.count() >= 1); + + assert_eq!(<P as GetStorageVersion>::on_chain_storage_version(), 1); +} + +fn log_migration(stage: &str, storage_prefix: &[u8], old_pallet_name: &str, new_pallet_name: &str) { + log::info!( + target: "runtime::session_historical", + "{} prefix of storage '{}': '{}' ==> '{}'", + stage, + str::from_utf8(storage_prefix).unwrap_or("<Invalid UTF8>"), + old_pallet_name, + new_pallet_name, + ); +} diff --git a/substrate/frame/session/src/tests.rs b/substrate/frame/session/src/tests.rs index 308ed7c5e548777fcc900c7dba94f991c1b77051..cc0606edf499db17f5527147898fa4b516a7fc2f 100644 --- a/substrate/frame/session/src/tests.rs +++ b/substrate/frame/session/src/tests.rs @@ -453,3 +453,30 @@ fn upgrade_keys() { } }) } + +#[cfg(feature = "historical")] +#[test] +fn test_migration_v1() { + use crate::{ + historical::{HistoricalSessions, StoredRange}, + mock::Historical, + }; + use frame_support::traits::PalletInfoAccess; + + new_test_ext().execute_with(|| { + assert!(<HistoricalSessions<Test>>::iter_values().count() > 0); + assert!(<StoredRange<Test>>::exists()); + + let old_pallet = "Session"; + let new_pallet = <Historical as PalletInfoAccess>::name(); + frame_support::storage::migration::move_pallet( + new_pallet.as_bytes(), + old_pallet.as_bytes(), + ); + StorageVersion::new(0).put::<Historical>(); + + crate::migrations::v1::pre_migrate::<Test, Historical>(); + crate::migrations::v1::migrate::<Test, Historical>(); + crate::migrations::v1::post_migrate::<Test, Historical>(); + }); +} diff --git a/substrate/frame/staking/src/mock.rs b/substrate/frame/staking/src/mock.rs index 95d397359f8d69ad22b2d78908cb756cddae9610..e5a3e49033934c573c826844a3c910eeb2a6317c 100644 --- a/substrate/frame/staking/src/mock.rs +++ b/substrate/frame/staking/src/mock.rs @@ -96,6 +96,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>}, Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>}, Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>}, + Historical: pallet_session::historical::{Pallet, Storage}, BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>}, } );