diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock
index f8c6bab59d162dc290c4514f70fa0e8a162acfe7..1002d1c894e1609db027f532fbf6a2cc1e195f5f 100644
--- a/substrate/Cargo.lock
+++ b/substrate/Cargo.lock
@@ -4318,20 +4318,6 @@ dependencies = [
"sp-std",
]
-[[package]]
-name = "pallet-benchmark"
-version = "2.0.0-rc6"
-dependencies = [
- "frame-benchmarking",
- "frame-support",
- "frame-system",
- "parity-scale-codec",
- "serde",
- "sp-io",
- "sp-runtime",
- "sp-std",
-]
-
[[package]]
name = "pallet-collective"
version = "2.0.0-rc6"
diff --git a/substrate/Cargo.toml b/substrate/Cargo.toml
index e69a95f1e4a646e68730ff087545f99b26bd83c1..99b1d418a5153c19ab2dc3a39d233c43fc7c8b7c 100644
--- a/substrate/Cargo.toml
+++ b/substrate/Cargo.toml
@@ -66,7 +66,6 @@ members = [
"frame/babe",
"frame/balances",
"frame/benchmarking",
- "frame/benchmark",
"frame/collective",
"frame/contracts",
"frame/contracts/rpc",
diff --git a/substrate/frame/benchmark/Cargo.toml b/substrate/frame/benchmark/Cargo.toml
deleted file mode 100644
index f731ebcbacf54583dcc19695243adda3c00b94c3..0000000000000000000000000000000000000000
--- a/substrate/frame/benchmark/Cargo.toml
+++ /dev/null
@@ -1,36 +0,0 @@
-[package]
-name = "pallet-benchmark"
-version = "2.0.0-rc6"
-authors = ["Parity Technologies <admin@parity.io>"]
-edition = "2018"
-license = "Apache-2.0"
-homepage = "https://substrate.dev"
-repository = "https://github.com/paritytech/substrate/"
-description = "Patterns to benchmark in a FRAME runtime."
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]
-
-[dependencies]
-serde = { version = "1.0.101", optional = true }
-codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
-sp-std = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/std" }
-sp-io = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/io" }
-sp-runtime = { version = "2.0.0-rc6", default-features = false, path = "../../primitives/runtime" }
-frame-support = { version = "2.0.0-rc6", default-features = false, path = "../support" }
-frame-system = { version = "2.0.0-rc6", default-features = false, path = "../system" }
-frame-benchmarking = { version = "2.0.0-rc6", default-features = false, path = "../benchmarking", optional = true }
-
-[features]
-default = ["std"]
-std = [
- "serde",
- "codec/std",
- "sp-std/std",
- "sp-io/std",
- "sp-runtime/std",
- "frame-support/std",
- "frame-system/std",
- "frame-benchmarking/std",
-]
-runtime-benchmarks = ["frame-benchmarking"]
diff --git a/substrate/frame/benchmark/README.md b/substrate/frame/benchmark/README.md
deleted file mode 100644
index e00e11292e1432fd26b397b5c80d9f0627e9554f..0000000000000000000000000000000000000000
--- a/substrate/frame/benchmark/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-A pallet that contains common runtime patterns in an isolated manner.
-This pallet is **not** meant to be used in a production blockchain, just
-for benchmarking and testing purposes.
-
-License: Apache-2.0
\ No newline at end of file
diff --git a/substrate/frame/benchmark/src/benchmarking.rs b/substrate/frame/benchmark/src/benchmarking.rs
deleted file mode 100644
index ddf3df9eaad4cb5a367724a5b2b1508359856bc4..0000000000000000000000000000000000000000
--- a/substrate/frame/benchmark/src/benchmarking.rs
+++ /dev/null
@@ -1,134 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) 2020 Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: Apache-2.0
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Benchmarks for common FRAME Pallet operations.
-
-#![cfg(feature = "runtime-benchmarks")]
-
-use super::*;
-
-use frame_system::RawOrigin;
-use sp_std::prelude::*;
-use frame_benchmarking::{benchmarks, account};
-
-use crate::Module as Benchmark;
-
-const SEED: u32 = 0;
-
-benchmarks! {
- _ {
- let m in 1 .. 1000 => {
- let origin = RawOrigin::Signed(account("member", m, SEED));
- Benchmark::<T>::add_member_list(origin.into())?
- };
- let i in 1 .. 1000 => {
- MyMap::insert(i, i);
- };
- let d in 1 .. 1000 => {
- for i in 0..d {
- for j in 0..100 {
- MyDoubleMap::insert(i, j, d);
- }
- }
- };
- }
-
- add_member_list {
- let m in ...;
- }: _(RawOrigin::Signed(account("member", m + 1, SEED)))
-
- append_member_list {
- let m in ...;
- }: _(RawOrigin::Signed(account("member", m + 1, SEED)))
-
- read_value {
- let n in 1 .. 1000;
- MyValue::put(n);
- }: _(RawOrigin::Signed(account("user", 0, SEED)), n)
-
- put_value {
- let n in 1 .. 1000;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), n)
-
- exists_value {
- let n in 1 .. 1000;
- MyValue::put(n);
- }: _(RawOrigin::Signed(account("user", 0, SEED)), n)
-
- remove_value {
- let i in ...;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), i)
-
- read_map {
- let i in ...;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), i)
-
- insert_map {
- let n in 1 .. 1000;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), n)
-
- contains_key_map {
- let i in ...;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), i)
-
- remove_prefix {
- let d in ...;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), d)
-
- do_nothing {
- let n in 1 .. 1000;
- }: _(RawOrigin::Signed(account("user", 0, SEED)), n)
-
- encode_accounts {
- let a in 1 .. 1000;
- let mut accounts = Vec::new();
- for _ in 0..a {
- accounts.push(account::<T::AccountId>("encode", a, SEED));
- }
- }: _(RawOrigin::Signed(account("user", 0, SEED)), accounts)
-
- decode_accounts {
- let a in 1 .. 1000;
- let mut accounts = Vec::new();
- for _ in 0..a {
- accounts.push(account::<T::AccountId>("encode", a, SEED));
- }
- let bytes = accounts.encode();
- }: _(RawOrigin::Signed(account("user", 0, SEED)), bytes)
-
- // Custom implementation to handle benchmarking of storage recalculation.
- // Puts `repeat` number of items into random storage keys, and then times how
- // long it takes to recalculate the storage root.
- storage_root {
- let z in 0 .. 10000;
- }: {
- for index in 0 .. z {
- let random = (index).using_encoded(sp_io::hashing::blake2_256);
- sp_io::storage::set(&random, &random);
- }
- }
-
- // Custom implementation to handle benchmarking of calling a host function.
- // Will check how long it takes to call `current_time()`.
- current_time {
- let z in 0 .. 1000;
- }: {
- for _ in 0 .. z {
- let _ = frame_benchmarking::benchmarking::current_time();
- }
- }
-}
diff --git a/substrate/frame/benchmark/src/lib.rs b/substrate/frame/benchmark/src/lib.rs
deleted file mode 100644
index 422272f817c7a2f75c2b991a40a01929b4dc48b2..0000000000000000000000000000000000000000
--- a/substrate/frame/benchmark/src/lib.rs
+++ /dev/null
@@ -1,191 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) 2020 Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: Apache-2.0
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! A pallet that contains common runtime patterns in an isolated manner.
-//! This pallet is **not** meant to be used in a production blockchain, just
-//! for benchmarking and testing purposes.
-
-#![cfg_attr(not(feature = "std"), no_std)]
-
-use frame_support::{decl_module, decl_storage, decl_event, decl_error};
-use frame_support::traits::Currency;
-use frame_system::{self as system, ensure_signed};
-use codec::{Encode, Decode};
-use sp_std::prelude::Vec;
-
-mod benchmarking;
-
-/// Type alias for currency balance.
-pub type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
-
-/// The pallet's configuration trait.
-pub trait Trait: system::Trait {
- type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
- type Currency: Currency<Self::AccountId>;
-}
-
-// This pallet's storage items.
-decl_storage! {
- trait Store for Module<T: Trait> as Benchmark {
- MyMemberList: Vec<T::AccountId>;
- MyMemberMap: map hasher(blake2_128_concat) T::AccountId => bool;
- MyValue: u32;
- MyMap: map hasher(twox_64_concat) u32 => u32;
- MyDoubleMap: double_map hasher(twox_64_concat) u32, hasher(identity) u32 => u32;
- }
-}
-
-// The pallet's events
-decl_event!(
- pub enum Event<T> where AccountId = <T as system::Trait>::AccountId {
- Dummy(u32, AccountId),
- }
-);
-
-// The pallet's errors
-decl_error! {
- pub enum Error for Module<T: Trait> {
- }
-}
-
-// The pallet's dispatchable functions.
-decl_module! {
- /// The module declaration.
- pub struct Module<T: Trait> for enum Call where origin: T::Origin {
- type Error = Error<T>;
-
- fn deposit_event() = default;
-
- /// Do nothing.
- #[weight = 0]
- pub fn do_nothing(_origin, input: u32) {
- if input > 0 {
- return Ok(());
- }
- }
-
- /// Read a value from storage value `repeat` number of times.
- /// Note the first `get()` read here will pull from the underlying
- /// storage database, however, the `repeat` calls will all pull from the
- /// storage overlay cache. You must consider this when analyzing the
- /// results of the benchmark.
- #[weight = 0]
- pub fn read_value(_origin, repeat: u32) {
- for _ in 0..repeat {
- MyValue::get();
- }
- }
-
- /// Put a value into a storage value.
- #[weight = 0]
- pub fn put_value(_origin, repeat: u32) {
- for r in 0..repeat {
- MyValue::put(r);
- }
- }
-
- /// Read a value from storage `repeat` number of times.
- /// Note the first `exists()` read here will pull from the underlying
- /// storage database, however, the `repeat` calls will all pull from the
- /// storage overlay cache. You must consider this when analyzing the
- /// results of the benchmark.
- #[weight = 0]
- pub fn exists_value(_origin, repeat: u32) {
- for _ in 0..repeat {
- MyValue::exists();
- }
- }
-
- /// Remove a value from storage `repeat` number of times.
- #[weight = 0]
- pub fn remove_value(_origin, repeat: u32) {
- for r in 0..repeat {
- MyMap::remove(r);
- }
- }
-
- /// Read a value from storage map `repeat` number of times.
- #[weight = 0]
- pub fn read_map(_origin, repeat: u32) {
- for r in 0..repeat {
- MyMap::get(r);
- }
- }
-
- /// Insert a value into a map.
- #[weight = 0]
- pub fn insert_map(_origin, repeat: u32) {
- for r in 0..repeat {
- MyMap::insert(r, r);
- }
- }
-
- /// Check is a map contains a value `repeat` number of times.
- #[weight = 0]
- pub fn contains_key_map(_origin, repeat: u32) {
- for r in 0..repeat {
- MyMap::contains_key(r);
- }
- }
-
- /// Read a value from storage `repeat` number of times.
- #[weight = 0]
- pub fn remove_prefix(_origin, repeat: u32) {
- for r in 0..repeat {
- MyDoubleMap::remove_prefix(r);
- }
- }
-
- /// Add user to the list.
- #[weight = 0]
- pub fn add_member_list(origin) {
- let who = ensure_signed(origin)?;
- MyMemberList::<T>::mutate(|x| x.push(who));
- }
-
- /// Append user to the list.
- #[weight = 0]
- pub fn append_member_list(origin) {
- let who = ensure_signed(origin)?;
- MyMemberList::<T>::append(who);
- }
-
- /// Encode a vector of accounts to bytes.
- #[weight = 0]
- pub fn encode_accounts(_origin, accounts: Vec<T::AccountId>) {
- let bytes = accounts.encode();
-
- // In an attempt to tell the compiler not to optimize away this benchmark, we will use
- // the result of encoding the accounts.
- if bytes.is_empty() {
- frame_support::print("You are encoding zero accounts.");
- }
- }
-
- /// Decode bytes into a vector of accounts.
- #[weight = 0]
- pub fn decode_accounts(_origin, bytes: Vec<u8>) {
- let accounts: Vec<T::AccountId> = Decode::decode(&mut bytes.as_slice()).map_err(|_| "Could not decode")?;
-
- // In an attempt to tell the compiler not to optimize away this benchmark, we will use
- // the result of decoding the bytes.
- if accounts.is_empty() {
- frame_support::print("You are decoding zero bytes.");
- }
- }
- }
-}
diff --git a/substrate/frame/benchmarking/README.md b/substrate/frame/benchmarking/README.md
index 1e06135e345e7317585be196c13440c9a98e0cbb..bf4bf951aa2b201e331799c85b401e0ae0f643db 100644
--- a/substrate/frame/benchmarking/README.md
+++ b/substrate/frame/benchmarking/README.md
@@ -1,3 +1,188 @@
-Macro for benchmarking a FRAME runtime.
+# Substrate Runtime Benchmarking Framework
-License: Apache-2.0
\ No newline at end of file
+This crate contains a set of utilities that can be used to benchmark and weigh FRAME pallets that
+you develop for your Substrate Runtime.
+
+## Overview
+
+Substrate's FRAME framework allows you to develop custom logic for your blockchain that can be
+included in your runtime. This flexibility is key to help you design complex and interactive
+pallets, but without accurate weights assigned to dispatchables, your blockchain may become
+vulnerable to denial of service (DoS) attacks by malicious actors.
+
+The Substrate Runtime Benchmarking Framework is a tool you can use to mitigate DoS attacks against
+your blockchain network by benchmarking the computational resources required to execute different
+functions in the runtime, for example extrinsics, `on_initialize`, `verify_unsigned`, etc...
+
+The general philosophy behind the benchmarking system is: If your node can know ahead of time how
+long it will take to execute an extrinsic, it can safely make decisions to include or exclude that
+extrinsic based on its available resources. By doing this, it can keep the block production and
+import process running smoothly.
+
+To achieve this, we need to model how long it takes to run each function in the runtime by:
+
+* Creating custom benchmarking logic that executes a specific code path of a function.
+* Executing the benchmark in the Wasm execution environment, on a specific set of hardware, with a
+ custom runtime configuration, etc...
+* Executing the benchmark across controlled ranges of possible values that may affect the result of
+ the benchmark (called "components").
+* Executing the benchmark multiple times at each point in order to isolate and remove outliers.
+* Using the results of the benchmark to create a linear model of the function across its components.
+
+With this linear model, we are able to estimate ahead of time how long it takes to execute some
+logic, and thus make informed decisions without actually spending any significant resources at
+runtime.
+
+Note that we assume that all extrinsics are assumed to be of linear complexity, which is why we are
+able to always fit them to a linear model. Quadratic or higher complexity functions are, in general,
+considered to be dangerous to the runtime as the weight of these functions may explode as the
+runtime state or input becomes too complex.
+
+The benchmarking framework comes with the following tools:
+
+* [A set of macros](./src/lib.rs) (`benchmarks!`, `add_benchmark!`, etc...) to make it easy to
+ write, test, and add runtime benchmarks.
+* [A set of linear regression analysis functions](./src/analysis.rs) for processing benchmark data.
+* [A CLI extension](../../utils/benchmarking-cli/) to make it easy to execute benchmarks on your
+ node.
+
+The end-to-end benchmarking pipeline is disabled by default when compiling a node. If you want to
+run benchmarks, you need to enable it by compiling with a Rust feature flag `runtime-benchmarks`.
+More details about this below.
+
+### Weight
+
+Substrate represents computational resources using a generic unit of measurement called "Weight". It
+defines 10^12 Weight as 1 second of computation on the physical machine used for benchmarking. This
+means that the weight of a function may change based on the specific hardware used to benchmark the
+runtime functions.
+
+By modeling the expected weight of each runtime function, the blockchain is able to calculate how
+many transactions or system level functions it will be able to execute within a certain period of
+time. Often, the limiting factor for a blockchain is the fixed block production time for the
+network.
+
+Within FRAME, each dispatchable function must have a `#[weight]` annotation with a function that can
+return the expected weight for the worst case scenario execution of that function given its inputs.
+This benchmarking framework will result in a file that automatically generates those formulas for
+you, which you can then use in your pallet.
+
+## Writing Benchmarks
+
+Writing a runtime benchmark is much like writing a unit test for your pallet. It needs to be
+carefully crafted to execute a certain logical path in your code. In tests you want to check for
+various success and failure conditions, but with benchmarks you specifically look for the **most
+computationally heavy** path, a.k.a the "worst case scenario".
+
+This means that if there are certain storage items or runtime state that may affect the complexity
+of the function, for example triggering more iterations in a `for` loop, to get an accurate result,
+you must set up your benchmark to trigger this.
+
+It may be that there are multiple paths your function can go down, and it is not clear which one is
+the heaviest. In this case, you should just create a benchmark for each scenario! You may find that
+there are paths in your code where complexity may become unbounded depending on user input. This may
+be a hint that you should enforce sane boundaries for how a user can use your pallet. For example:
+limiting the number of elements in a vector, limiting the number of iterations in a `for` loop,
+etc...
+
+Examples of end-to-end benchmarks can be found in the [pallets provided by Substrate](../), and the
+specific details on how to use the `benchmarks!` macro can be found in [its
+documentation](./src/lib.rs).
+
+## Testing Benchmarks
+
+You can test your benchmarks using the same test runtime that you created for your pallet's unit
+tests. By creating your benchmarks in the `benchmarks!` macro, it automatically generates test
+functions for you:
+
+```rust
+fn test_benchmark_[benchmark_name]<T>::() -> Result<(), &'static str>
+```
+
+Simply add these functions to a unit test and ensure that the result of the function is `Ok(())`.
+
+> **Note:** If your test runtime and production runtime have different configurations, you may get
+different results when testing your benchmark and actually running it.
+
+In general, benchmarks returning `Ok(())` is all you need to check for since it signals the executed
+extrinsic has completed successfully. However, you can optionally include a `verify` block with your
+benchmark, which can additionally verify any final conditions, such as the final state of your
+runtime.
+
+These additional `verify` blocks will not affect the results of your final benchmarking process.
+
+To run the tests, you need to enable the `runtime-benchmarks` feature flag. This may also mean you
+need to move into your node's binary folder. For example, with the Substrate repository, this is how
+you would test the Balances pallet's benchmarks:
+
+```bash
+cd bin/node/cli
+cargo test -p pallet-balances --features runtime-benchmarks
+```
+
+## Adding Benchmarks
+
+The benchmarks included with each pallet are not automatically added to your node. To actually
+execute these benchmarks, you need to implement the `frame_benchmarking::Benchmark` trait. You can
+see an example of how to do this in the [included Substrate
+node](../../bin/node/runtime/src/lib.rs).
+
+Assuming there are already some benchmarks set up on your node, you just need to add another
+instance of the `add_benchmark!` macro:
+
+```rust
+/// configuration for running benchmarks
+/// | name of your pallet's crate (as imported)
+/// v v
+add_benchmark!(params, batches, pallet_balances, Balances);
+/// ^ ^
+/// where all benchmark results are saved |
+/// the `struct` created for your pallet by `construct_runtime!`
+```
+
+Once you have done this, you will need to compile your node binary with the `runtime-benchmarks`
+feature flag:
+
+```bash
+cd bin/node/cli
+cargo build --release --features runtime-benchmarks
+```
+
+## Running Benchmarks
+
+Finally, once you have a node binary with benchmarks enabled, you need to execute your various
+benchmarks.
+
+You can get a list of the available benchmarks by running:
+
+```bash
+./target/release/substrate benchmark --chain dev --pallet "*" --extrinsic "*" --repeat 0
+```
+
+Then you can run a benchmark like so:
+
+```bash
+./target/release/substrate benchmark \
+ --chain dev \ # Configurable Chain Spec
+ --execution=wasm \ # Always test with Wasm
+ --wasm-execution=compiled \ # Always used `wasm-time`
+ --pallet pallet_balances \ # Select the pallet
+ --extrinsic transfer \ # Select the extrinsic
+ --steps 50 \ # Number of samples across component ranges
+ --repeat 20 \ # Number of times we repeat a benchmark
+ --output \ # Output benchmark results into a Rust file
+```
+
+This will output a file `pallet_name.rs` which implements the `WeightInfo` trait you should include
+in your pallet. Each blockchain should generate their own benchmark file with their custom
+implementation of the `WeightInfo` trait. This means that you will be able to use these modular
+Substrate pallets while still keeping your network safe for your specific configuration and
+requirements.
+
+To get a full list of available options when running benchmarks, run:
+
+```bash
+./target/release/substrate benchmark --help
+```
+
+License: Apache-2.0
diff --git a/substrate/frame/benchmarking/src/analysis.rs b/substrate/frame/benchmarking/src/analysis.rs
index 6963d84ee614e20f94906099244d439092d86358..dafb4a74b669fef3c90330acae1ae32e27d1712f 100644
--- a/substrate/frame/benchmarking/src/analysis.rs
+++ b/substrate/frame/benchmarking/src/analysis.rs
@@ -15,7 +15,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//! Tools for analysing the benchmark results.
+//! Tools for analyzing the benchmark results.
use std::collections::BTreeMap;
use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionModel};
diff --git a/substrate/frame/benchmarking/src/lib.rs b/substrate/frame/benchmarking/src/lib.rs
index 6a457d2a5e912763a2bec4d1babbdb874ae2b5ec..b189cdb6e705e9ed8dc7ce68421c50f271b404eb 100644
--- a/substrate/frame/benchmarking/src/lib.rs
+++ b/substrate/frame/benchmarking/src/lib.rs
@@ -158,7 +158,7 @@ pub use sp_storage::TrackedStorageKey;
/// }
/// ```
///
-/// These `verify` blocks will not execute when running your actual benchmarks!
+/// These `verify` blocks will not affect your benchmark results!
///
/// You can construct benchmark tests like so:
///