Newer
Older
// This file is part of Substrate.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Remote Externalities
//!
//! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate
//! based chain, or a local state snapshot file.
use async_recursion::async_recursion;
Liam Aharon
committed
use codec::{Compact, Decode, Encode};
Liam Aharon
committed
use indicatif::{ProgressBar, ProgressStyle};
use jsonrpsee::{
core::params::ArrayParams,
http_client::{HttpClient, HttpClientBuilder},
};
use log::*;
use serde::de::DeserializeOwned;
storage::{
well_known_keys::{is_default_child_storage_key, DEFAULT_CHILD_STORAGE_KEY_PREFIX},
ChildInfo, ChildType, PrefixedStorageKey, StorageData, StorageKey,
},
Liam Aharon
committed
H256,
pub use sp_io::TestExternalities;
use sp_runtime::{traits::Block as BlockT, StateVersion};
Liam Aharon
committed
use spinners::{Spinner, Spinners};
Liam Aharon
committed
cmp::max,
Liam Aharon
committed
time::{Duration, Instant},
use substrate_rpc_client::{rpc_params, BatchRequestBuilder, ChainApi, ClientT, StateApi};
use tokio_retry::{strategy::FixedInterval, Retry};
type KeyValue = (StorageKey, StorageData);
type TopKeyValues = Vec<KeyValue>;
type ChildKeyValues = Vec<(ChildInfo, Vec<KeyValue>)>;
Liam Aharon
committed
type SnapshotVersion = Compact<u16>;
const LOG_TARGET: &str = "remote-ext";
const DEFAULT_HTTP_ENDPOINT: &str = "https://rpc.polkadot.io:443";
Liam Aharon
committed
const SNAPSHOT_VERSION: SnapshotVersion = Compact(2);
/// The snapshot that we store on disk.
#[derive(Decode, Encode)]
struct Snapshot<B: BlockT> {
Liam Aharon
committed
snapshot_version: SnapshotVersion,
state_version: StateVersion,
block_hash: B::Hash,
Liam Aharon
committed
// <Vec<Key, (Value, MemoryDbRefCount)>>
raw_storage: Vec<(H256, (Vec<u8>, i32))>,
Liam Aharon
committed
storage_root: H256,
Liam Aharon
committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
impl<B: BlockT> Snapshot<B> {
pub fn new(
state_version: StateVersion,
block_hash: B::Hash,
raw_storage: Vec<(H256, (Vec<u8>, i32))>,
storage_root: H256,
) -> Self {
Self {
snapshot_version: SNAPSHOT_VERSION,
state_version,
block_hash,
raw_storage,
storage_root,
}
}
fn load(path: &PathBuf) -> Result<Snapshot<B>, &'static str> {
let bytes = fs::read(path).map_err(|_| "fs::read failed.")?;
// The first item in the SCALE encoded struct bytes is the snapshot version. We decode and
// check that first, before proceeding to decode the rest of the snapshot.
let maybe_version: Result<SnapshotVersion, _> = Decode::decode(&mut &*bytes);
match maybe_version {
Ok(snapshot_version) => {
if snapshot_version != SNAPSHOT_VERSION {
return Err(
"Unsupported snapshot version detected. Please create a new snapshot.",
)
}
match Decode::decode(&mut &*bytes) {
Ok(snapshot) => return Ok(snapshot),
Err(_) => Err("Decode failed"),
}
},
Err(_) => Err("Decode failed"),
}
}
}
/// An externalities that acts exactly the same as [`sp_io::TestExternalities`] but has a few extra
/// bits and pieces to it, and can be loaded remotely.
pub struct RemoteExternalities<B: BlockT> {
/// The inner externalities.
pub inner_ext: TestExternalities,
/// The block hash it which we created this externality env.
pub block_hash: B::Hash,
}
impl<B: BlockT> Deref for RemoteExternalities<B> {
type Target = TestExternalities;
fn deref(&self) -> &Self::Target {
&self.inner_ext
}
}
impl<B: BlockT> DerefMut for RemoteExternalities<B> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner_ext
}
}
/// The execution mode.
#[derive(Clone)]
/// Online. Potentially writes to a snapshot file.
Online(OnlineConfig<B>),
/// Offline. Uses a state snapshot file and needs not any client config.
/// Prefer using a snapshot file if it exists, else use a remote server.
OfflineOrElseOnline(OfflineConfig, OnlineConfig<B>),
impl<B: BlockT> Default for Mode<B> {
fn default() -> Self {
Mode::Online(OnlineConfig::default())
}
}
/// Configuration of the offline execution.
/// A state snapshot config must be present.
#[derive(Clone)]
pub struct OfflineConfig {
/// The configuration of the state snapshot file to use. It must be present.
pub state_snapshot: SnapshotConfig,
/// Description of the transport protocol (for online execution).
#[derive(Debug, Clone)]
pub enum Transport {
/// Use the `URI` to open a new WebSocket connection.
Uri(String),
/// Use HTTP connection.
Liam Aharon
committed
RemoteClient(HttpClient),
impl Transport {
fn as_client(&self) -> Option<&HttpClient> {
match self {
Self::RemoteClient(client) => Some(client),
_ => None,
}
}
// Build an HttpClient from a URI.
async fn init(&mut self) -> Result<(), &'static str> {
if let Self::Uri(uri) = self {
log::debug!(target: LOG_TARGET, "initializing remote client to {:?}", uri);
// If we have a ws uri, try to convert it to an http uri.
// We use an HTTP client rather than WS because WS starts to choke with "accumulated
// message length exceeds maximum" errors after processing ~10k keys when fetching
// from a node running a default configuration.
let uri = if uri.starts_with("ws://") {
let uri = uri.replace("ws://", "http://");
log::info!(target: LOG_TARGET, "replacing ws:// in uri with http://: {:?} (ws is currently unstable for fetching remote storage, for more see https://github.com/paritytech/jsonrpsee/issues/1086)", uri);
uri
} else if uri.starts_with("wss://") {
let uri = uri.replace("wss://", "https://");
log::info!(target: LOG_TARGET, "replacing wss:// in uri with https://: {:?} (ws is currently unstable for fetching remote storage, for more see https://github.com/paritytech/jsonrpsee/issues/1086)", uri);
uri
} else {
uri.clone()
};
let http_client = HttpClientBuilder::default()
.max_request_body_size(u32::MAX)
.request_timeout(std::time::Duration::from_secs(60 * 5))
.build(uri)
Loading full blame...