diff --git a/.config/lychee.toml b/.config/lychee.toml index 733b77ec0cff9e616ecc0f851d9a3ed5e8574636..ad6a0ef75545f640c042c451d3efc0ffe26d7daf 100644 --- a/.config/lychee.toml +++ b/.config/lychee.toml @@ -32,6 +32,7 @@ exclude = [ "https://github.com/paritytech/polkadot-sdk/substrate/frame/timestamp", "https://github.com/paritytech/substrate/frame/fast-unstake", "https://github.com/zkcrypto/bls12_381/blob/e224ad4ea1babfc582ccd751c2bf128611d10936/src/test-data/mod.rs", + "https://polkadot-try-runtime-node.parity-chains.parity.io/", "https://polkadot.network/the-path-of-a-parachain-block/", "https://research.web3.foundation/en/latest/polkadot/BABE/Babe/#6-practical-results", "https://research.web3.foundation/en/latest/polkadot/NPoS/3.%20Balancing.html", diff --git a/prdoc/pr_4349.prdoc b/prdoc/pr_4349.prdoc new file mode 100644 index 0000000000000000000000000000000000000000..fdc9e816e1b9630cb505f7d8270850ad9450c57e --- /dev/null +++ b/prdoc/pr_4349.prdoc @@ -0,0 +1,9 @@ +title: "Store Header in RemoteExt Snapshot" + +doc: + - audience: Runtime Dev + description: Replaces the block hash in the RemoteExt snapshot with the block header. + +crates: + - name: frame-remote-externalities + bump: major diff --git a/substrate/utils/frame/remote-externalities/src/lib.rs b/substrate/utils/frame/remote-externalities/src/lib.rs index 58cb901470c17df20f877ed96930b9be05f7d88d..201b5e176f34bec0ed66ed53427424e904d2621a 100644 --- a/substrate/utils/frame/remote-externalities/src/lib.rs +++ b/substrate/utils/frame/remote-externalities/src/lib.rs @@ -36,7 +36,7 @@ use sp_core::{ }, }; use sp_runtime::{ - traits::{Block as BlockT, Hash, HashingFor}, + traits::{Block as BlockT, HashingFor}, StateVersion, }; use sp_state_machine::TestExternalities; @@ -58,37 +58,39 @@ type ChildKeyValues = Vec<(ChildInfo, Vec<KeyValue>)>; type SnapshotVersion = Compact<u16>; const LOG_TARGET: &str = "remote-ext"; -const DEFAULT_HTTP_ENDPOINT: &str = "https://rpc.polkadot.io:443"; -const SNAPSHOT_VERSION: SnapshotVersion = Compact(3); +const DEFAULT_HTTP_ENDPOINT: &str = "https://polkadot-try-runtime-node.parity-chains.parity.io:443"; +const SNAPSHOT_VERSION: SnapshotVersion = Compact(4); /// The snapshot that we store on disk. #[derive(Decode, Encode)] -struct Snapshot<H> { +struct Snapshot<B: BlockT> { snapshot_version: SnapshotVersion, state_version: StateVersion, - block_hash: H, // <Vec<Key, (Value, MemoryDbRefCount)>> raw_storage: Vec<(Vec<u8>, (Vec<u8>, i32))>, - storage_root: H, + // The storage root of the state. This may vary from the storage root in the header, if not the + // entire state was fetched. + storage_root: B::Hash, + header: B::Header, } -impl<H: Decode> Snapshot<H> { +impl<B: BlockT> Snapshot<B> { pub fn new( state_version: StateVersion, - block_hash: H, raw_storage: Vec<(Vec<u8>, (Vec<u8>, i32))>, - storage_root: H, + storage_root: B::Hash, + header: B::Header, ) -> Self { Self { snapshot_version: SNAPSHOT_VERSION, state_version, - block_hash, raw_storage, storage_root, + header, } } - fn load(path: &PathBuf) -> Result<Snapshot<H>, &'static str> { + fn load(path: &PathBuf) -> Result<Snapshot<B>, &'static str> { let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; // The first item in the SCALE encoded struct bytes is the snapshot version. We decode and // check that first, before proceeding to decode the rest of the snapshot. @@ -105,21 +107,21 @@ impl<H: Decode> Snapshot<H> { /// An externalities that acts exactly the same as [`sp_io::TestExternalities`] but has a few extra /// bits and pieces to it, and can be loaded remotely. -pub struct RemoteExternalities<H: Hash> { +pub struct RemoteExternalities<B: BlockT> { /// The inner externalities. - pub inner_ext: TestExternalities<H>, - /// The block hash with which we created this externality env. - pub block_hash: H::Out, + pub inner_ext: TestExternalities<HashingFor<B>>, + /// The block header which we created this externality env. + pub header: B::Header, } -impl<H: Hash> Deref for RemoteExternalities<H> { - type Target = TestExternalities<H>; +impl<B: BlockT> Deref for RemoteExternalities<B> { + type Target = TestExternalities<HashingFor<B>>; fn deref(&self) -> &Self::Target { &self.inner_ext } } -impl<H: Hash> DerefMut for RemoteExternalities<H> { +impl<B: BlockT> DerefMut for RemoteExternalities<B> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner_ext } @@ -859,7 +861,7 @@ where } } -impl<B: BlockT + DeserializeOwned> Builder<B> +impl<B: BlockT> Builder<B> where B::Hash: DeserializeOwned, B::Header: DeserializeOwned, @@ -1030,6 +1032,21 @@ where Ok(()) } + async fn load_header(&self) -> Result<B::Header, &'static str> { + let retry_strategy = + FixedInterval::new(Self::KEYS_PAGE_RETRY_INTERVAL).take(Self::MAX_RETRIES); + let get_header_closure = || { + ChainApi::<(), _, B::Header, ()>::header( + self.as_online().rpc_client(), + Some(self.as_online().at_expected()), + ) + }; + Retry::spawn(retry_strategy, get_header_closure) + .await + .map_err(|_| "Failed to fetch header for block from network")? + .ok_or("Network returned None block header") + } + /// Load the data from a remote server. The main code path is calling into `load_top_remote` and /// `load_child_remote`. /// @@ -1058,13 +1075,11 @@ where // If we need to save a snapshot, save the raw storage and root hash to the snapshot. if let Some(path) = self.as_online().state_snapshot.clone().map(|c| c.path) { let (raw_storage, storage_root) = pending_ext.into_raw_snapshot(); - let snapshot = Snapshot::<B::Hash>::new( + let snapshot = Snapshot::<B>::new( state_version, - self.as_online() - .at - .expect("set to `Some` in `init_remote_client`; must be called before; qed"), raw_storage.clone(), storage_root, + self.load_header().await?, ); let encoded = snapshot.encode(); log::info!( @@ -1086,22 +1101,21 @@ where Ok(pending_ext) } - async fn do_load_remote(&mut self) -> Result<RemoteExternalities<HashingFor<B>>, &'static str> { + async fn do_load_remote(&mut self) -> Result<RemoteExternalities<B>, &'static str> { self.init_remote_client().await?; - let block_hash = self.as_online().at_expected(); let inner_ext = self.load_remote_and_maybe_save().await?; - Ok(RemoteExternalities { block_hash, inner_ext }) + Ok(RemoteExternalities { header: self.load_header().await?, inner_ext }) } fn do_load_offline( &mut self, config: OfflineConfig, - ) -> Result<RemoteExternalities<HashingFor<B>>, &'static str> { + ) -> Result<RemoteExternalities<B>, &'static str> { let mut sp = Spinner::with_timer(Spinners::Dots, "Loading snapshot...".into()); let start = Instant::now(); info!(target: LOG_TARGET, "Loading snapshot from {:?}", &config.state_snapshot.path); - let Snapshot { snapshot_version: _, block_hash, state_version, raw_storage, storage_root } = - Snapshot::<B::Hash>::load(&config.state_snapshot.path)?; + let Snapshot { snapshot_version: _, header, state_version, raw_storage, storage_root } = + Snapshot::<B>::load(&config.state_snapshot.path)?; let inner_ext = TestExternalities::from_raw_snapshot( raw_storage, @@ -1110,12 +1124,10 @@ where ); sp.stop_with_message(format!("✅ Loaded snapshot ({:.2}s)", start.elapsed().as_secs_f32())); - Ok(RemoteExternalities { inner_ext, block_hash }) + Ok(RemoteExternalities { inner_ext, header }) } - pub(crate) async fn pre_build( - mut self, - ) -> Result<RemoteExternalities<HashingFor<B>>, &'static str> { + pub(crate) async fn pre_build(mut self) -> Result<RemoteExternalities<B>, &'static str> { let mut ext = match self.mode.clone() { Mode::Offline(config) => self.do_load_offline(config)?, Mode::Online(_) => self.do_load_remote().await?, @@ -1154,7 +1166,7 @@ where } // Public methods -impl<B: BlockT + DeserializeOwned> Builder<B> +impl<B: BlockT> Builder<B> where B::Hash: DeserializeOwned, B::Header: DeserializeOwned, @@ -1191,7 +1203,7 @@ where self } - pub async fn build(self) -> Result<RemoteExternalities<HashingFor<B>>, &'static str> { + pub async fn build(self) -> Result<RemoteExternalities<B>, &'static str> { let mut ext = self.pre_build().await?; ext.commit_all().unwrap(); @@ -1226,7 +1238,7 @@ mod tests { init_logger(); Builder::<Block>::new() .mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig::new("test_data/proxy_test"), + state_snapshot: SnapshotConfig::new("test_data/test.snap"), })) .build() .await @@ -1241,7 +1253,7 @@ mod tests { // get the first key from the snapshot file. let some_key = Builder::<Block>::new() .mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig::new("test_data/proxy_test"), + state_snapshot: SnapshotConfig::new("test_data/test.snap"), })) .build() .await @@ -1255,7 +1267,7 @@ mod tests { Builder::<Block>::new() .mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig::new("test_data/proxy_test"), + state_snapshot: SnapshotConfig::new("test_data/test.snap"), })) .blacklist_hashed_key(&some_key) .build() @@ -1341,7 +1353,7 @@ mod remote_tests { .await .unwrap(); - assert_eq!(ext.block_hash, cached_ext.block_hash); + assert_eq!(ext.header.hash(), cached_ext.header.hash()); } #[tokio::test] diff --git a/substrate/utils/frame/remote-externalities/test_data/proxy_test b/substrate/utils/frame/remote-externalities/test_data/proxy_test deleted file mode 100644 index f0b1b4f5af40bc8a159c9ee250bee7849cababae..0000000000000000000000000000000000000000 Binary files a/substrate/utils/frame/remote-externalities/test_data/proxy_test and /dev/null differ diff --git a/substrate/utils/frame/remote-externalities/test_data/test.snap b/substrate/utils/frame/remote-externalities/test_data/test.snap new file mode 100644 index 0000000000000000000000000000000000000000..28f2012d0f2a1b6a63ee8825deaf70257625279a Binary files /dev/null and b/substrate/utils/frame/remote-externalities/test_data/test.snap differ