diff --git a/.gitignore b/.gitignore index aadaa13912c19dbc056245784442d8b4c9de2a82..6398c09fe796278130978c2a8598ee9270399388 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ rls*.log *.rej **/wip/*.stderr .local +**/hfuzz_target/ +**/hfuzz_workspace/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 962ecbe3e476880510c295cd93df19fdfc93247f..2c141c51d129617d06ad73eeecb7d652a1a876fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ stages: - test - build + - post-build-test - publish - kubernetes - flaming-fir @@ -31,7 +32,7 @@ stages: variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 - CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_JOB_NAME}" + CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" CARGO_INCREMENTAL: 0 CI_SERVER_NAME: "GitLab CI" @@ -39,7 +40,7 @@ variables: ARCH: "x86_64" # FIXME set to release CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.10" - CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" + CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder sp-arithmetic-fuzzer" .collect-artifacts: &collect-artifacts @@ -127,11 +128,13 @@ check-line-width: allow_failure: true -check-polkadot: + + +check-polkadot-companion-build: stage: build <<: *docker-env script: - - ./.maintain/gitlab/check_polkadot.sh + - ./.maintain/gitlab/check_polkadot_companion_build.sh interruptible: true allow_failure: true @@ -145,6 +148,20 @@ cargo-audit: - cargo audit allow_failure: true +cargo-deny: + stage: test + <<: *docker-env + script: + - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml + after_script: + - echo "___The complete log is in the artifacts___" + - cargo deny check -c .maintain/deny.toml 2> deny.log + artifacts: + name: $CI_COMMIT_SHORT_SHA + expire_in: 3 days + when: always + paths: + - deny.log cargo-check-benches: stage: test @@ -176,17 +193,17 @@ test-linux-stable: &test-linux variables: - $DEPLOY_TAG script: - - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked | + - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |& tee output.log - sccache -s - after_script: - - echo "___Collecting warnings for check_warnings job___" + - echo "____Test job successful, checking for warnings____" - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_warnings.log + - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then + cat ${CI_COMMIT_SHORT_SHA}_warnings.log; + exit 1; + else + echo "___No warnings___"; + fi test-dependency-rules: stage: test @@ -270,7 +287,7 @@ test-runtime-benchmarks: - $DEPLOY_TAG script: - cd bin/node/cli - - BUILD_DUMMY_WASM_BINARY=1 time cargo check --verbose --features runtime-benchmarks + - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features runtime-benchmarks - sccache -s test-linux-stable-int: @@ -314,7 +331,7 @@ check-web-wasm: - time cargo build --target=wasm32-unknown-unknown -p sp-consensus - time cargo build --target=wasm32-unknown-unknown -p sc-telemetry # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way. - - time cargo build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown + - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown -Z features=itarget - sccache -s test-full-crypto-feature: @@ -400,24 +417,15 @@ build-rust-doc-release: - echo "" > ./crate-docs/index.html - sccache -s -check_warnings: - stage: build - <<: *docker-env - except: - variables: - - $DEPLOY_TAG - variables: - GIT_STRATEGY: none - needs: - - job: test-linux-stable - artifacts: true +check-polkadot-companion-status: + stage: post-build-test + image: parity/tools:latest + <<: *kubernetes-build + only: + - /^[0-9]+$/ script: - - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then - cat ${CI_COMMIT_SHORT_SHA}_warnings.log; - exit 1; - else - echo "___No warnings___"; - fi + - ./.maintain/gitlab/check_polkadot_companion_status.sh + trigger-contracts-ci: diff --git a/.maintain/deny.toml b/.maintain/deny.toml new file mode 100644 index 0000000000000000000000000000000000000000..8cc7635d5049be795e994b0ba740d9273b2f520f --- /dev/null +++ b/.maintain/deny.toml @@ -0,0 +1,193 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #{ triple = "x86_64-unknown-linux-musl" }, + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory database is cloned/fetched into +db-path = "~/.cargo/advisory-db" +# The url of the advisory database to use +db-url = "https://github.com/rustsec/advisory-db" +# The lint level for security vulnerabilities +vulnerability = "deny" +# The lint level for unmaintained crates +unmaintained = "warn" +# The lint level for crates that have been yanked from their source registry +yanked = "warn" +# The lint level for crates with security notices. Note that as of +# 2019-12-17 there are no security notice advisories in +# https://github.com/rustsec/advisory-db +notice = "warn" +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", +] +# Threshold for security vulnerabilities, any vulnerability with a CVSS score +# lower than the range specified will be ignored. Note that ignored advisories +# will still output a note when they are encountered. +# * None - CVSS Score 0.0 +# * Low - CVSS Score 0.1 - 3.9 +# * Medium - CVSS Score 4.0 - 6.9 +# * High - CVSS Score 7.0 - 8.9 +# * Critical - CVSS Score 9.0 - 10.0 +#severity-threshold = + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# The lint level for crates which do not have a detectable license +unlicensed = "deny" +# List of explictly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.7 short identifier (+ optional exception)]. +allow = [ + #"MIT", + #"Apache-2.0", + #"Apache-2.0 WITH LLVM-exception", +] +# List of explictly disallowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.7 short identifier (+ optional exception)]. +deny = [ + #"Nokia", +] +# Lint level for licenses considered copyleft +copyleft = "allow" +# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses +# * both - The license will be approved if it is both OSI-approved *AND* FSF +# * either - The license will be approved if it is either OSI-approved *OR* FSF +# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF +# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved +# * neither - This predicate is ignored and the default lint level is used +allow-osi-fsf-free = "either" +# Lint level used when no other predicates are matched +# 1. License isn't in the allow or deny lists +# 2. License isn't copyleft +# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" +default = "deny" +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.9 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], name = "adler32", version = "*" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +[[licenses.clarify]] +# The name of the crate the clarification applies to +name = "ring" +# THe optional version constraint for the crate +#version = "*" +# The SPDX expression for the license requirements of the crate +expression = "OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +license-files = [ + # Each entry is a crate relative path, and the (opaque) hash of its contents + { path = "LICENSE", hash = 0xbd0eed23 } +] +[[licenses.clarify]] +name = "webpki" +expression = "ISC" +license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "lowest-version" +# List of crates that are allowed. Use with care! +allow = [ + #{ name = "ansi_term", version = "=0.11.0" }, +] +# List of crates to deny +deny = [ + { name = "parity-util-mem", version = "<0.6" } + # Each entry the name of a crate and a version range. If version is + # not specified, all versions will be matched. +] +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + #{ name = "ansi_term", version = "=0.11.0" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite +skip-tree = [ + #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "deny" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] diff --git a/.maintain/gitlab/check_polkadot.sh b/.maintain/gitlab/check_polkadot_companion_build.sh similarity index 50% rename from .maintain/gitlab/check_polkadot.sh rename to .maintain/gitlab/check_polkadot_companion_build.sh index 6087d86cdea475dcb5ac703879b22d91ea25e212..eb4c6440f46c9fcafbd6a9ea9d78102518a4e619 100755 --- a/.maintain/gitlab/check_polkadot.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -3,9 +3,11 @@ # check if a pr is compatible with polkadot companion pr or master if not # available # -# mark companion pr in the body of the polkadot pr like +# to override one that was just mentioned mark companion pr in the body of the +# polkadot pr like # # polkadot companion: paritytech/polkadot#567 +# github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" @@ -20,22 +22,24 @@ boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; don boldcat <<-EOT -check_polkadot -============== +check_polkadot_companion_build +============================== this job checks if there is a string in the description of the pr like polkadot companion: paritytech/polkadot#567 -it will then run cargo check from this polkadot's branch with substrate code -from this pull request. +it will then run cargo check from this polkadot's branch with substrate code +from this pull request. in absence of that string it will check if a polkadot +pr is mentioned and will use the last one instead. if none of the above can be +found it will check if polkadot has a branch of the exact same name than the +substrate's branch. if it can't find anything, it will uses master instead EOT - SUBSTRATE_PATH=$(pwd) # Clone the current Polkadot master branch into ./polkadot. @@ -48,17 +52,43 @@ cd polkadot if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null then boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" + + pr_data_file="$(mktemp)" # get the last reference to a pr in polkadot - comppr="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \ - | sed -n -r 's;^[[:space:]]+"body":[[:space:]]+".*polkadot companion: paritytech/polkadot#([0-9]+).*"[^"]+$;\1;p;$!d')" - if [ "${comppr}" ] + curl -sSL -H "${github_header}" -o "${pr_data_file}" \ + "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}" + + pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")" + + pr_companion="$(echo "${pr_body}" | sed -n -r \ + -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \ + -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ + | tail -n 1)" + + if [ -z "${pr_companion}" ] + then + pr_companion="$(echo "${pr_body}" | sed -n -r \ + -e 's;^.*paritytech/polkadot/#([0-9]+).*$;\1;p' \ + -e 's;^.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ + | tail -n 1)" + fi + + if [ "${pr_companion}" ] then - boldprint "companion pr specified: #${comppr}" - git fetch --depth 1 origin refs/pull/${comppr}/head:pr/${comppr} - git checkout pr/${comppr} + boldprint "companion pr specified/detected: #${pr_companion}" + git fetch --depth 1 origin refs/pull/${pr_companion}/head:pr/${pr_companion} + git checkout pr/${pr_companion} else - boldprint "no companion pr declared - building polkadot:master" + pr_ref="$(grep -Po '"ref"\s*:\s*"\K(?!master)[^"]*' "${pr_data_file}")" + if git fetch --depth 1 origin "$pr_ref":branch/"$pr_ref" 2>/dev/null + then + boldprint "companion branch detected: $pr_ref" + git checkout branch/"$pr_ref" + else + boldprint "no companion branch found - building polkadot:master" + fi fi + rm -f "${pr_data_file}" else boldprint "this is not a pull request - building polkadot:master" fi diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh new file mode 100755 index 0000000000000000000000000000000000000000..5387e68f25cbba284433b8946f20f6eacfc76b9f --- /dev/null +++ b/.maintain/gitlab/check_polkadot_companion_status.sh @@ -0,0 +1,100 @@ +#!/bin/sh +# +# check for a polkadot companion pr and ensure it has approvals and is +# mergeable +# + +github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" +github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls" +# use github api v3 in order to access the data without authentication +github_header="Accept: application/vnd.github.v3+json" + +boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } +boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } + + + +boldcat <<-EOT + + +check_polkadot_companion_status +=============================== + +this job checks if there is a string in the description of the pr like + +polkadot companion: paritytech/polkadot#567 + +or any other polkadot pr is mentioned in this pr's description and checks its +status. + + +EOT + + +if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ] +then + boldprint "this doesn't seem to be a pull request" + exit 1 +fi + +boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" + +pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \ + | sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')" + +# get companion if explicitly specified +pr_companion="$(echo "${pr_body}" | sed -n -r \ + -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \ + -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ + | tail -n 1)" + +# get companion mentioned in the description +if [ -z "${pr_companion}" ] +then + pr_companion="$(echo "${pr_body}" | sed -n -r \ + 's;^.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ + | tail -n 1)" +fi + +if [ -z "${pr_companion}" ] +then + boldprint "no companion pr found" + exit 0 +fi + +boldprint "companion pr: #${pr_companion}" + +# check the status of that pull request - needs to be +# mergable and approved + +curl -H "${github_header}" -sS -o companion_pr.json \ + ${github_api_polkadot_pull_url}/${pr_companion} + +if jq -e .merged < companion_pr.json >/dev/null +then + boldprint "polkadot pr #${pr_companion} already merged" + exit 0 +fi + +if jq -e '.mergeable' < companion_pr.json >/dev/null +then + boldprint "polkadot pr #${pr_companion} mergeable" +else + boldprint "polkadot pr #${pr_companion} not mergeable" + exit 1 +fi + +curl -H "${github_header}" -sS -o companion_pr_reviews.json \ + ${github_api_polkadot_pull_url}/${pr_companion}/reviews + +if [ -n "$(jq -r -e '.[].state | select(. == "CHANGES_REQUESTED")' < companion_pr_reviews.json)" ] && \ + [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ] +then + boldprint "polkadot pr #${pr_companion} not APPROVED" + exit 1 +fi + +boldprint "polkadot pr #${pr_companion} state APPROVED" +exit 0 + + diff --git a/.maintain/node-template-release/Cargo.toml b/.maintain/node-template-release/Cargo.toml index 606def19bb99f1002058d1d868181cbc9c0fbb56..dd3166d58ddf463580474a02ea70efdcdbf00228 100644 --- a/.maintain/node-template-release/Cargo.toml +++ b/.maintain/node-template-release/Cargo.toml @@ -16,3 +16,6 @@ git2 = "0.8" flate2 = "1.0" [workspace] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/Cargo.lock b/Cargo.lock index 4f1e5a4c566d80e5f80965722a12427a289b2c31..de2b174826e02f89887efd4b17258dea9722d9a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,14 +70,13 @@ dependencies = [ [[package]] name = "alga" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658f9468113d34781f6ca9d014d174c74b73de870f1e0e3ad32079bbab253b19" +checksum = "4f823d037a7ec6ea2197046bafd4ae150e6bc36f9ca347404f46a46823fa84f2" dependencies = [ "approx", - "libm", "num-complex", - "num-traits", + "num-traits 0.2.11", ] [[package]] @@ -100,9 +99,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" [[package]] name = "app_dirs" @@ -122,14 +121,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" dependencies = [ - "num-traits", + "num-traits 0.2.11", ] +[[package]] +name = "arbitrary" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed" + [[package]] name = "arc-swap" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" +checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" [[package]] name = "arrayref" @@ -167,21 +172,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] name = "assert_cmd" -version = "0.12.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6283bac8dd7226470d491bc4737816fea4ca1fba7a2847f2e9097fd6bfb4624c" +checksum = "c88b9ca26f9c16ec830350d309397e74ee9abdfd8eb1f71cb6ecc71a3fc818da" dependencies = [ "doc-comment", - "escargot", "predicates", "predicates-core", "predicates-tree", + "wait-timeout", ] [[package]] @@ -205,7 +210,7 @@ dependencies = [ "futures-io", "futures-timer 2.0.2", "kv-log-macro", - "log 0.4.8", + "log", "memchr", "mio", "mio-uds", @@ -263,9 +268,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" dependencies = [ "backtrace-sys", "cfg-if", @@ -275,9 +280,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca797db0057bae1a7aa2eef3283a874695455cecf08a43bfb8507ee0ebc1ed69" +checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" dependencies = [ "cc", "libc", @@ -289,16 +294,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" -[[package]] -name = "base64" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" -dependencies = [ - "byteorder 1.3.4", - "safemem", -] - [[package]] name = "base64" version = "0.10.1" @@ -338,10 +333,10 @@ dependencies = [ "env_logger 0.7.1", "lazy_static", "lazycell", - "log 0.4.8", + "log", "peeking_take_while", "proc-macro2", - "quote", + "quote 1.0.3", "regex", "rustc-hash", "shlex", @@ -362,9 +357,13 @@ checksum = "5da9b3d9f6f585199287a473f4f8dfab6566cf827d15c00c219f53c645687ead" [[package]] name = "bitvec" -version = "0.15.2" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" +checksum = "41262f11d771fd4a61aa3ce019fca363b4b6c282fca9da2a31186d3965a47a5c" +dependencies = [ + "either", + "radium", +] [[package]] name = "blake2" @@ -399,6 +398,17 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "blake2s_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9e07352b829279624ceb7c64adb4f585dacdb81d35cafae81139ccd617cf44" +dependencies = [ + "arrayref", + "arrayvec 0.5.1", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -451,9 +461,9 @@ checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae" [[package]] name = "bstr" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502ae1441a0a5adb8fbd38a5955a6416b9493e92b465de5e4a9bde6a539c2c48" +checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" dependencies = [ "lazy_static", "memchr", @@ -472,9 +482,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.2.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" +checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" [[package]] name = "byte-slice-cast" @@ -597,16 +607,16 @@ checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" dependencies = [ "js-sys", "num-integer", - "num-traits", + "num-traits 0.2.11", "time", "wasm-bindgen", ] [[package]] name = "clang-sys" -version = "0.29.2" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92986241798376849e1a007827041fed9bb36195822c2049d18e174420e0534" +checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a" dependencies = [ "glob 0.3.0", "libc", @@ -671,7 +681,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7871d2947441b0fdd8e2bd1ce2a2f75304f896582c0d572162d48290683c48" dependencies = [ - "log 0.4.8", + "log", "web-sys", ] @@ -738,7 +748,7 @@ dependencies = [ "cranelift-codegen-shared", "cranelift-entity", "gimli", - "log 0.4.8", + "log", "serde", "smallvec 1.2.0", "target-lexicon", @@ -777,7 +787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "518344698fa6c976d853319218415fdfb4f1bc6b42d0b2e2df652e55dff1f778" dependencies = [ "cranelift-codegen", - "log 0.4.8", + "log", "smallvec 1.2.0", "target-lexicon", ] @@ -802,7 +812,7 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "log 0.4.8", + "log", "serde", "thiserror", "wasmparser", @@ -831,7 +841,7 @@ dependencies = [ "itertools", "lazy_static", "libc", - "num-traits", + "num-traits 0.2.11", "rand_core 0.3.1", "rand_os", "rand_xoshiro", @@ -857,7 +867,7 @@ dependencies = [ "csv", "itertools", "lazy_static", - "num-traits", + "num-traits 0.2.11", "oorandom", "plotters", "rayon", @@ -1000,8 +1010,8 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" dependencies = [ - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1045,13 +1055,13 @@ checksum = "11c0346158a19b3627234e15596f5e465c360fcdb97d817bcb255e0510f5a788" [[package]] name = "derive_more" -version = "0.99.3" +version = "0.99.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" +checksum = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1103,9 +1113,9 @@ dependencies = [ [[package]] name = "doc-comment" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "807e5847c39ad6a11eac66de492ed1406f76a260eb8656e8740cad9eabc69c27" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "ed25519-dalek" @@ -1125,6 +1135,17 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +[[package]] +name = "enum-primitive-derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b90e520ec62c1864c8c78d637acbfe8baf5f63240f2fb8165b8325c07812dd" +dependencies = [ + "num-traits 0.1.43", + "quote 0.3.15", + "syn 0.11.11", +] + [[package]] name = "enumflags2" version = "0.6.2" @@ -1141,8 +1162,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1153,7 +1174,7 @@ checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" dependencies = [ "atty", "humantime", - "log 0.4.8", + "log", "regex", "termcolor", ] @@ -1166,7 +1187,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", - "log 0.4.8", + "log", "regex", "termcolor", ] @@ -1179,18 +1200,18 @@ checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" [[package]] name = "erased-serde" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7d80305c9bd8cd78e3c753eb9fb110f83621e5211f1a3afffcc812b104daf9" +checksum = "d88b6d1705e16a4d62e05ea61cc0496c2bd190f4fa8e5c1f11ce747be6bcf3d1" dependencies = [ "serde", ] [[package]] name = "errno" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a071601ed01b988f896ab14b95e67335d1eeb50190932a1320f7fe3cadc84e" +checksum = "b480f641ccf0faf324e20c1d3e53d81b7484c698b42ea677f6907ae4db195371" dependencies = [ "errno-dragonfly", "libc", @@ -1207,18 +1228,6 @@ dependencies = [ "libc", ] -[[package]] -name = "escargot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74cf96bec282dcdb07099f7e31d9fed323bca9435a09aba7b6d99b7617bca96d" -dependencies = [ - "lazy_static", - "log 0.4.8", - "serde", - "serde_json", -] - [[package]] name = "ethbloom" version = "0.9.0" @@ -1310,7 +1319,7 @@ dependencies = [ "anyhow", "goblin", "indexmap", - "log 0.4.8", + "log", "scroll", "string-interner", "target-lexicon", @@ -1334,8 +1343,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", "synstructure", ] @@ -1367,7 +1376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8505b75b31ef7285168dd237c4a7db3c1f3e0927e7d314e670bc98e854272fe9" dependencies = [ "env_logger 0.6.2", - "log 0.4.8", + "log", ] [[package]] @@ -1378,8 +1387,8 @@ checksum = "024517816630be5204eba201e8d1d405042b1255a5e0e3f298b054fc24d59e1d" dependencies = [ "futures 0.3.4", "futures-timer 2.0.2", - "log 0.4.8", - "num-traits", + "log", + "num-traits 0.2.11", "parity-scale-codec", "parking_lot 0.9.0", "rand 0.6.5", @@ -1405,9 +1414,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "flate2" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", @@ -1422,21 +1431,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "fork-tree" version = "2.0.0-alpha.5" @@ -1452,6 +1446,7 @@ dependencies = [ "frame-system", "linregress", "parity-scale-codec", + "paste", "sp-api", "sp-io", "sp-runtime", @@ -1515,7 +1510,7 @@ dependencies = [ "frame-support-procedural", "frame-system", "impl-trait-for-tuples", - "log 0.4.8", + "log", "once_cell", "parity-scale-codec", "paste", @@ -1537,8 +1532,8 @@ version = "2.0.0-alpha.5" dependencies = [ "frame-support-procedural-tools", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1548,8 +1543,8 @@ dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1557,8 +1552,8 @@ name = "frame-support-procedural-tools-derive" version = "2.0.0-alpha.5" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1724,7 +1719,7 @@ dependencies = [ "futures 0.1.29", "futures 0.3.4", "lazy_static", - "log 0.4.8", + "log", "parking_lot 0.9.0", "pin-project", "serde", @@ -1757,8 +1752,8 @@ checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -1807,7 +1802,6 @@ dependencies = [ "proc-macro-hack", "proc-macro-nested", "slab", - "tokio-io", ] [[package]] @@ -1911,14 +1905,14 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "globset" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2" +checksum = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120" dependencies = [ "aho-corasick", "bstr", "fnv", - "log 0.4.8", + "log", "regex", ] @@ -1941,7 +1935,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081214398d39e4bd7f2c1975f0488ed04614ffdd976c6fc7a0708278552c0da" dependencies = [ - "log 0.4.8", + "log", "plain", "scroll", ] @@ -1958,7 +1952,7 @@ dependencies = [ "futures 0.1.29", "http 0.1.21", "indexmap", - "log 0.4.8", + "log", "slab", "string", "tokio-io", @@ -1966,18 +1960,18 @@ dependencies = [ [[package]] name = "h2" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5c295d1c0c68e4e42003d75f908f5e16a1edd1cbe0b0d02e4dc2006a384f47" +checksum = "377038bf3c89d18d6ca1431e7a5027194fbd724ca10592b9487ede5e8e144f42" dependencies = [ "bytes 0.5.4", "fnv", "futures-core", "futures-sink", "futures-util", - "http 0.2.0", + "http 0.2.1", "indexmap", - "log 0.4.8", + "log", "slab", "tokio 0.2.13", "tokio-util", @@ -2019,9 +2013,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1010591b26bbfe835e9faeabeb11866061cc7dcebffd56ad7d0942d0e61aefd8" +checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" dependencies = [ "libc", ] @@ -2072,6 +2066,17 @@ dependencies = [ "hmac", ] +[[package]] +name = "honggfuzz" +version = "0.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3de2c3273ef7735df1c5a72128ca85b1d20105b9aac643cdfd7a6e581311150" +dependencies = [ + "arbitrary", + "lazy_static", + "memmap", +] + [[package]] name = "http" version = "0.1.21" @@ -2085,9 +2090,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ "bytes 0.5.4", "fnv", @@ -2113,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ "bytes 0.5.4", - "http 0.2.0", + "http 0.2.1", ] [[package]] @@ -2131,25 +2136,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "hyper" -version = "0.10.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" -dependencies = [ - "base64 0.9.3", - "httparse", - "language-tags", - "log 0.3.9", - "mime", - "num_cpus", - "time", - "traitobject", - "typeable", - "unicase 1.4.2", - "url 1.7.2", -] - [[package]] name = "hyper" version = "0.12.35" @@ -2165,7 +2151,7 @@ dependencies = [ "httparse", "iovec", "itoa", - "log 0.4.8", + "log", "net2", "rustc_version", "time", @@ -2182,20 +2168,20 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5" +checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e" dependencies = [ "bytes 0.5.4", "futures-channel", "futures-core", "futures-util", - "h2 0.2.2", - "http 0.2.0", + "h2 0.2.4", + "http 0.2.1", "http-body 0.3.1", "httparse", "itoa", - "log 0.4.8", + "log", "net2", "pin-project", "time", @@ -2213,8 +2199,8 @@ dependencies = [ "bytes 0.5.4", "ct-logs", "futures-util", - "hyper 0.13.3", - "log 0.4.8", + "hyper 0.13.4", + "log", "rustls 0.17.0", "rustls-native-certs", "tokio 0.2.13", @@ -2222,19 +2208,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "hyper-tls" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.29", - "hyper 0.12.35", - "native-tls", - "tokio-io", -] - [[package]] name = "idna" version = "0.1.5" @@ -2300,8 +2273,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -2325,6 +2298,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77" +[[package]] +name = "intervalier" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14200459dc2319eb13708aed1c1efb8307e0e0e801e7282476939492e1492631" +dependencies = [ + "futures 0.3.4", + "futures-timer 2.0.2", +] + [[package]] name = "iovec" version = "0.1.4" @@ -2336,9 +2319,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a859057dc563d1388c1e816f98a1892629075fc046ed06e845b883bb8b2916fb" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "itertools" @@ -2366,9 +2349,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb931d43e71f560c81badb0191596562bafad2be06a3f9025b845c847c60df5" +checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" dependencies = [ "wasm-bindgen", ] @@ -2382,15 +2365,12 @@ dependencies = [ "failure", "futures 0.1.29", "hyper 0.12.35", - "hyper-tls", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.8", + "log", "serde", "serde_json", - "tokio 0.1.22", "url 1.7.2", - "websocket", ] [[package]] @@ -2400,7 +2380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe3b688648f1ef5d5072229e2d672ecb92cbff7d1c79bcf3fd5898f3f3df0970" dependencies = [ "futures 0.1.29", - "log 0.4.8", + "log", "serde", "serde_derive", "serde_json", @@ -2423,8 +2403,8 @@ checksum = "8609af8f63b626e8e211f52441fcdb6ec54f1a446606b10d5c89ae9bf8a20058" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -2436,10 +2416,10 @@ dependencies = [ "hyper 0.12.35", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.8", + "log", "net2", "parking_lot 0.10.0", - "unicase 2.6.0", + "unicase", ] [[package]] @@ -2449,7 +2429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b31c9b90731276fdd24d896f31bb10aecf2e5151733364ae81123186643d939" dependencies = [ "jsonrpc-core", - "log 0.4.8", + "log", "parking_lot 0.10.0", "serde", ] @@ -2464,10 +2444,10 @@ dependencies = [ "globset", "jsonrpc-core", "lazy_static", - "log 0.4.8", + "log", "tokio 0.1.22", "tokio-codec", - "unicase 2.6.0", + "unicase", ] [[package]] @@ -2478,7 +2458,7 @@ checksum = "b94e5773b2ae66e0e02c80775ce6bbba6f15d5bb47c14ec36a36fcf94f8df851" dependencies = [ "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.8", + "log", "parking_lot 0.10.0", "slab", "ws", @@ -2517,7 +2497,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c54d9f465d530a752e6ebdc217e081a7a614b48cb200f6f0aee21ba6bc9aabb" dependencies = [ - "log 0.4.8", + "log", ] [[package]] @@ -2550,7 +2530,7 @@ dependencies = [ "fs-swap", "interleaved-ordered", "kvdb", - "log 0.4.8", + "log", "num_cpus", "owning_ref", "parity-util-mem", @@ -2570,19 +2550,13 @@ dependencies = [ "js-sys", "kvdb", "kvdb-memorydb", - "log 0.4.8", + "log", "parity-util-mem", "send_wrapper 0.3.0", "wasm-bindgen", "web-sys", ] -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" - [[package]] name = "lazy_static" version = "1.4.0" @@ -2603,9 +2577,21 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" +checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" + +[[package]] +name = "libflate" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9135df43b1f5d0e333385cb6e7897ecd1a43d7d11b91ac003f4d2c2d2401fdd" +dependencies = [ + "adler32", + "crc32fast", + "rle-decode-fast", + "take_mut", +] [[package]] name = "libloading" @@ -2619,15 +2605,15 @@ dependencies = [ [[package]] name = "libm" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba17ee9cac4bb89de5812159877d9b4f0a993bf41697a5a875940cd1eb71f24" +checksum = "8a261244b8d7ff58f5d62ffa33589eb1ba7733a1dfee0902ad9fdfe62ada7009" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2653,8 +2639,8 @@ dependencies = [ "libp2p-wasm-ext", "libp2p-websocket", "libp2p-yamux", - "parity-multiaddr", - "parity-multihash", + "multihash", + "parity-multiaddr 0.8.0", "parking_lot 0.10.0", "pin-project", "smallvec 1.2.0", @@ -2663,22 +2649,23 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.16.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b874594c4b29de1a29f27871feba8e6cd13aa54a8a1e8f8c7cf3dfac5ca287c" +checksum = "1cfe1412f2afe1366a2661abd211bb1a27ee6a664d799071282f4fba997c6858" dependencies = [ "asn1_der", "bs58", "ed25519-dalek", + "either", "fnv", "futures 0.3.4", "futures-timer 3.0.2", "lazy_static", "libsecp256k1", - "log 0.4.8", + "log", + "multihash", "multistream-select", - "parity-multiaddr", - "parity-multihash", + "parity-multiaddr 0.8.0", "parking_lot 0.10.0", "pin-project", "prost", @@ -2696,19 +2683,19 @@ dependencies = [ [[package]] name = "libp2p-core-derive" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d472e9d522f588805c77801de10b957be84e10f019ca5f869fa1825b15ea9b" +checksum = "a0eeb25d5f152a826eac57c7d1cc3de10d72dc4051e90fe4c0cd139f07a069a3" dependencies = [ - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] name = "libp2p-deflate" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e25004d4d9837b44b22c5f1a69be1724a5168fef6cff1716b5176a972c3aa62" +checksum = "136fcef31e3247f51946c3ebefb94d0798c4c8aae78bc59cb7431b220b5330cf" dependencies = [ "flate2", "futures 0.3.4", @@ -2717,20 +2704,20 @@ dependencies = [ [[package]] name = "libp2p-dns" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b99e552f9939b606eb4b59f7f64d9b01e3f96752f47e350fc3c5fc646ed3f649" +checksum = "647178f8683bf868f7f14d5e5718dbdc2445b9f6b24ce99da96cecd7c5d2d1a6" dependencies = [ "futures 0.3.4", "libp2p-core", - "log 0.4.8", + "log", ] [[package]] name = "libp2p-floodsub" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3234f12e44f9a50351a9807b97fe7de11eb9ae4482370392ba10da6dc90722" +checksum = "34c8dee172fd1630caf91a427d601d6a8d24c8cfcbcf7d5c09c9a1f3b4bbebc2" dependencies = [ "cuckoofilter", "fnv", @@ -2745,9 +2732,9 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d46cb3e0841bd951cbf4feae56cdc081e6347836a644fb260c3ec554149b4006" +checksum = "0042a2156fb6264bda9def93070e411dfaddf8c57c4b2d63634190d296458c76" dependencies = [ "base64 0.11.0", "byteorder 1.3.4", @@ -2757,7 +2744,7 @@ dependencies = [ "futures_codec", "libp2p-core", "libp2p-swarm", - "log 0.4.8", + "log", "lru", "prost", "prost-build", @@ -2770,14 +2757,14 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfeb935a9bd41263e4f3a24b988e9f4a044f3ae89ac284e83c17fe2f84e0d66b" +checksum = "04efa011cda5232648b5aa50bd80be7ba0a695d682b01aa46b65e5be5ece0605" dependencies = [ "futures 0.3.4", "libp2p-core", "libp2p-swarm", - "log 0.4.8", + "log", "prost", "prost-build", "smallvec 1.2.0", @@ -2786,9 +2773,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464dc8412978d40f0286be72ed9ab5e0e1386a4a06e7f174526739b5c3c1f041" +checksum = "97f4722d83af8fc0065cee7589a000b629961c1c11d90ba09f6685b3e123b9ae" dependencies = [ "arrayvec 0.5.1", "bytes 0.5.4", @@ -2798,8 +2785,8 @@ dependencies = [ "futures_codec", "libp2p-core", "libp2p-swarm", - "log 0.4.8", - "parity-multihash", + "log", + "multihash", "prost", "prost-build", "rand 0.7.3", @@ -2813,9 +2800,9 @@ dependencies = [ [[package]] name = "libp2p-mdns" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881fcfb360c2822db9f0e6bb6f89529621556ed9a8b038313414eda5107334de" +checksum = "b752276b3bea2fca1c291f43cefc8082d8a639bb8f9052cf5adc6accfcd7b44e" dependencies = [ "async-std", "data-encoding", @@ -2825,7 +2812,7 @@ dependencies = [ "lazy_static", "libp2p-core", "libp2p-swarm", - "log 0.4.8", + "log", "net2", "rand 0.7.3", "smallvec 1.2.0", @@ -2835,31 +2822,31 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8507b37ad0eed275efcde67a023c3d85af6c80768b193845b9288e848e1af95" +checksum = "0f317db8c062beecde87a8765ca03784e6f1a55daa5b9868bf60ebf9b9a2b92f" dependencies = [ "bytes 0.5.4", "fnv", "futures 0.3.4", "futures_codec", "libp2p-core", - "log 0.4.8", + "log", "parking_lot 0.10.0", "unsigned-varint", ] [[package]] name = "libp2p-noise" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15a8a3d71f898beb6f854c8aae27aa1d198e0d1f2e49412261c2d90ef39675a" +checksum = "98d3845f54288ff134dd78c131517bad8bc03965def6e6517efef03291d9b4d7" dependencies = [ "curve25519-dalek", "futures 0.3.4", "lazy_static", "libp2p-core", - "log 0.4.8", + "log", "prost", "prost-build", "rand 0.7.3", @@ -2872,14 +2859,14 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d22f2f228b3a828dca1cb8aa9fa331e0bc9c36510cb2c1916956e20dc85e8c" +checksum = "aa1cb80ccbedb91d9b980aafc6bf39dc7e4616a7e37c631a4e6ef62629567a13" dependencies = [ "futures 0.3.4", "libp2p-core", "libp2p-swarm", - "log 0.4.8", + "log", "rand 0.7.3", "void", "wasm-timer", @@ -2887,15 +2874,15 @@ dependencies = [ [[package]] name = "libp2p-plaintext" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56126a204d7b3382bac163143ff4125a14570b3ba76ba979103d1ae1abed1923" +checksum = "da16d35e3990cc5dc22c8d7ea4a2aa1c18f518491bb29c0c3498fb9a2d8e486e" dependencies = [ "bytes 0.5.4", "futures 0.3.4", "futures_codec", "libp2p-core", - "log 0.4.8", + "log", "prost", "prost-build", "rw-stream-sink", @@ -2905,12 +2892,12 @@ dependencies = [ [[package]] name = "libp2p-pnet" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b916938a8868f75180aeeffcc6a516a922d165e8fa2a90b57bad989d1ccbb57a" +checksum = "45d11e8c6d83e294ef3d7ff3a9f5a7aa5aa0c39c2d4991f2905c23c438c84526" dependencies = [ "futures 0.3.4", - "log 0.4.8", + "log", "pin-project", "rand 0.7.3", "salsa20", @@ -2919,9 +2906,9 @@ dependencies = [ [[package]] name = "libp2p-secio" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1219e9ecb4945d7331a05f5ffe96a1f6e28051bfa1223d4c60353c251de0354e" +checksum = "74130fa95effb780850ec790b7af777b893108d9b5983ab994b61d93d2eb0336" dependencies = [ "aes-ctr", "ctr", @@ -2930,7 +2917,7 @@ dependencies = [ "js-sys", "lazy_static", "libp2p-core", - "log 0.4.8", + "log", "parity-send-wrapper", "pin-project", "prost", @@ -2949,13 +2936,14 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "275471e7c0e88ae004660866cd54f603bd8bd1f4caef541a27f50dd8640c4d4c" +checksum = "a4ec53df8978a5d6d9dac243fb1e3adf004f8b8d28f72e6f2160df34d5f39564" dependencies = [ "futures 0.3.4", "libp2p-core", - "log 0.4.8", + "log", + "rand 0.7.3", "smallvec 1.2.0", "void", "wasm-timer", @@ -2963,9 +2951,9 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e80ad4e3535345f3d666554ce347d3100453775611c05c60786bf9a1747a10" +checksum = "e25c9d9c5448c189bba7ecdd1ca23800516281476e82810eff711ef04abaf9eb" dependencies = [ "async-std", "futures 0.3.4", @@ -2973,26 +2961,26 @@ dependencies = [ "get_if_addrs", "ipnet", "libp2p-core", - "log 0.4.8", + "log", ] [[package]] name = "libp2p-uds" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d329564a43da9d0e055a5b938633c4a8ceab1f59cec133fbc4647917c07341" +checksum = "d8dbcbe6567ea1b3c98ba4df5fd9d1b7c2bebbf50d46ceb0c2ce735c55af3f8d" dependencies = [ "async-std", "futures 0.3.4", "libp2p-core", - "log 0.4.8", + "log", ] [[package]] name = "libp2p-wasm-ext" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923581c055bc4b8c5f42d4ce5ef43e52fe5216f1ea4bc26476cb8a966ce6220b" +checksum = "076446cabb23b0d79d2375661d837a43cbda6719d88787f234e7661c33ef9554" dependencies = [ "futures 0.3.4", "js-sys", @@ -3004,16 +2992,16 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5351ca9eea122081c1c0f9323164d2918cac29b5a6bfe5054d4ba8ec9447cf42" +checksum = "a0117ed6a6f60114c107c1232a0890a8fe997013c7e1920b6f0c811e05d2fae7" dependencies = [ "async-tls", "bytes 0.5.4", "either", "futures 0.3.4", "libp2p-core", - "log 0.4.8", + "log", "quicksink", "rustls 0.16.0", "rw-stream-sink", @@ -3025,9 +3013,9 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dac30de24ccde0e67f363d71a125c587bbe6589503f664947e9b084b68a34f1" +checksum = "ee12c49426527908f81ffb6551b95f57149a8ea64f386bb7da3b123cdb9c01ba" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -3120,15 +3108,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -dependencies = [ - "log 0.4.8", -] - [[package]] name = "log" version = "0.4.8" @@ -3183,13 +3162,23 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.8", +] + [[package]] name = "memoffset" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" dependencies = [ - "rustc_version", + "autocfg 1.0.0", ] [[package]] @@ -3222,15 +3211,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "mime" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -dependencies = [ - "log 0.3.9", -] - [[package]] name = "miniz_oxide" version = "0.3.6" @@ -3252,7 +3232,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.8", + "log", "miow", "net2", "slab", @@ -3266,7 +3246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", - "log 0.4.8", + "log", "mio", "slab", ] @@ -3300,23 +3280,38 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +[[package]] +name = "multihash" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47fbc227f7e2b1cb701f95404579ecb2668abbdd3c7ef7a6cbb3cc0d3b236869" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "digest", + "sha-1", + "sha2", + "sha3", + "unsigned-varint", +] + [[package]] name = "multimap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97fbd5d00e0e37bfb10f433af8f5aaf631e739368dc9fc28286ca81ca4948dc" +checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" [[package]] name = "multistream-select" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f938ffe420493e77c8b6cbcc3f282283f68fc889c5dcbc8e51668d5f3a01ad94" +checksum = "74cdcf7cfb3402881e15a1f95116cb033d69b33c83d481e1234777f5ef0c3d2c" dependencies = [ "bytes 0.5.4", - "futures 0.1.29", - "log 0.4.8", + "futures 0.3.4", + "log", + "pin-project", "smallvec 1.2.0", - "tokio-io", "unsigned-varint", ] @@ -3332,7 +3327,7 @@ dependencies = [ "matrixmultiply", "num-complex", "num-rational", - "num-traits", + "num-traits 0.2.11", "rand 0.6.5", "typenum", ] @@ -3347,32 +3342,28 @@ dependencies = [ ] [[package]] -name = "native-tls" -version = "0.2.4" +name = "net2" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" dependencies = [ - "lazy_static", + "cfg-if", "libc", - "log 0.4.8", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", + "winapi 0.3.8", ] [[package]] -name = "net2" -version = "0.2.33" +name = "netstat2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "29449d242064c48d3057a194b049a2bdcccadda16faa18a91468677b44e8d422" dependencies = [ - "cfg-if", + "bitflags", + "byteorder 1.3.4", + "enum-primitive-derive", "libc", - "winapi 0.3.8", + "num-traits 0.2.11", + "thiserror", ] [[package]] @@ -3388,6 +3379,21 @@ dependencies = [ "void", ] +[[package]] +name = "node-bench" +version = "0.8.0-alpha.5" +dependencies = [ + "log", + "node-primitives", + "node-testing", + "sc-cli", + "sc-client-api", + "serde", + "serde_json", + "sp-runtime", + "structopt", +] + [[package]] name = "node-cli" version = "2.0.0-alpha.5" @@ -3399,7 +3405,7 @@ dependencies = [ "futures 0.3.4", "hex-literal", "jsonrpc-core", - "log 0.4.8", + "log", "nix", "node-executor", "node-inspect", @@ -3412,10 +3418,13 @@ dependencies = [ "pallet-contracts", "pallet-im-online", "pallet-indices", + "pallet-staking", "pallet-timestamp", "pallet-transaction-payment", "parity-scale-codec", + "platforms", "rand 0.7.3", + "regex", "sc-authority-discovery", "sc-basic-authorship", "sc-chain-spec", @@ -3454,7 +3463,6 @@ dependencies = [ "substrate-build-script-utils", "tempfile", "tracing", - "vergen", "wasm-bindgen", "wasm-bindgen-futures", ] @@ -3498,7 +3506,7 @@ name = "node-inspect" version = "0.8.0-alpha.5" dependencies = [ "derive_more", - "log 0.4.8", + "log", "parity-scale-codec", "sc-cli", "sc-client-api", @@ -3550,7 +3558,7 @@ dependencies = [ "futures 0.1.29", "hyper 0.12.35", "jsonrpc-core-client", - "log 0.4.8", + "log", "node-primitives", "sc-rpc", ] @@ -3583,8 +3591,10 @@ dependencies = [ "pallet-indices", "pallet-membership", "pallet-offences", + "pallet-offences-benchmarking", "pallet-randomness-collective-flip", "pallet-recovery", + "pallet-scheduler", "pallet-session", "pallet-session-benchmarking", "pallet-society", @@ -3622,7 +3632,7 @@ name = "node-template" version = "2.0.0-alpha.5" dependencies = [ "futures 0.3.4", - "log 0.4.8", + "log", "node-template-runtime", "sc-basic-authorship", "sc-cli", @@ -3643,7 +3653,6 @@ dependencies = [ "sp-transaction-pool", "structopt", "substrate-build-script-utils", - "vergen", ] [[package]] @@ -3686,7 +3695,7 @@ dependencies = [ "frame-support", "frame-system", "fs_extra", - "log 0.4.8", + "log", "node-executor", "node-primitives", "node-runtime", @@ -3728,7 +3737,7 @@ dependencies = [ name = "node-transaction-factory" version = "0.8.0-alpha.5" dependencies = [ - "log 0.4.8", + "log", "parity-scale-codec", "sc-block-builder", "sc-cli", @@ -3762,7 +3771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6" dependencies = [ "memchr", - "version_check 0.9.1", + "version_check", ] [[package]] @@ -3782,7 +3791,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg 1.0.0", "num-integer", - "num-traits", + "num-traits 0.2.11", ] [[package]] @@ -3792,7 +3801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ "autocfg 1.0.0", - "num-traits", + "num-traits 0.2.11", ] [[package]] @@ -3802,19 +3811,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" dependencies = [ "autocfg 1.0.0", - "num-traits", + "num-traits 0.2.11", ] [[package]] name = "num-rational" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg 1.0.0", "num-bigint", "num-integer", - "num-traits", + "num-traits 0.2.11", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.11", ] [[package]] @@ -3824,6 +3842,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" dependencies = [ "autocfg 1.0.0", + "libm", ] [[package]] @@ -3881,39 +3900,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" -[[package]] -name = "openssl" -version = "0.10.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973293749822d7dd6370d6da1e523b0d1db19f06c459134c658b2a4261378b52" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "lazy_static", - "libc", - "openssl-sys", -] - [[package]] name = "openssl-probe" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" -[[package]] -name = "openssl-sys" -version = "0.9.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1024c0a59774200a555087a6da3f253a9095a5f344e353b212ac4c8b8e450986" -dependencies = [ - "autocfg 1.0.0", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "output_vt100" version = "0.1.2" @@ -4008,12 +4000,9 @@ version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", - "hex-literal", - "lazy_static", "pallet-session", "pallet-timestamp", "parity-scale-codec", - "parking_lot 0.10.0", "serde", "sp-consensus-babe", "sp-consensus-vrf", @@ -4024,8 +4013,6 @@ dependencies = [ "sp-staking", "sp-std", "sp-timestamp", - "sp-version", - "substrate-test-runtime", ] [[package]] @@ -4062,6 +4049,7 @@ dependencies = [ name = "pallet-collective" version = "2.0.0-alpha.5" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "hex-literal", @@ -4142,10 +4130,12 @@ dependencies = [ name = "pallet-democracy" version = "2.0.0-alpha.5" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "hex-literal", "pallet-balances", + "pallet-scheduler", "parity-scale-codec", "serde", "sp-core", @@ -4179,6 +4169,7 @@ dependencies = [ "frame-system", "hex-literal", "pallet-balances", + "pallet-scheduler", "parity-scale-codec", "serde", "sp-core", @@ -4363,37 +4354,71 @@ dependencies = [ "frame-system", "pallet-balances", "parity-scale-codec", - "serde", - "sp-core", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-offences" +version = "2.0.0-alpha.5" +dependencies = [ + "frame-support", + "frame-system", + "pallet-balances", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", +] + +[[package]] +name = "pallet-offences-benchmarking" +version = "2.0.0-alpha.5" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-im-online", + "pallet-offences", + "pallet-session", + "pallet-staking", + "parity-scale-codec", "sp-io", "sp-runtime", + "sp-staking", "sp-std", ] [[package]] -name = "pallet-offences" +name = "pallet-randomness-collective-flip" version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", - "pallet-balances", "parity-scale-codec", - "serde", + "safe-mix", "sp-core", "sp-io", "sp-runtime", - "sp-staking", "sp-std", ] [[package]] -name = "pallet-randomness-collective-flip" +name = "pallet-recovery" version = "2.0.0-alpha.5" dependencies = [ + "enumflags2", "frame-support", "frame-system", + "pallet-balances", "parity-scale-codec", - "safe-mix", + "serde", "sp-core", "sp-io", "sp-runtime", @@ -4401,13 +4426,12 @@ dependencies = [ ] [[package]] -name = "pallet-recovery" +name = "pallet-scheduler" version = "2.0.0-alpha.5" dependencies = [ - "enumflags2", + "frame-benchmarking", "frame-support", "frame-system", - "pallet-balances", "parity-scale-codec", "serde", "sp-core", @@ -4456,9 +4480,17 @@ name = "pallet-session-benchmarking" version = "2.0.0-alpha.5" dependencies = [ "frame-benchmarking", + "frame-support", "frame-system", + "pallet-balances", "pallet-session", "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", "sp-runtime", "sp-std", ] @@ -4483,24 +4515,31 @@ dependencies = [ name = "pallet-staking" version = "2.0.0-alpha.5" dependencies = [ + "env_logger 0.7.1", "frame-benchmarking", "frame-support", "frame-system", + "hex", "pallet-authorship", "pallet-balances", + "pallet-indices", "pallet-session", "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", + "parking_lot 0.10.0", + "rand 0.7.3", "rand_chacha 0.2.2", "serde", + "sp-application-crypto", "sp-core", "sp-io", - "sp-keyring", "sp-phragmen", "sp-runtime", "sp-staking", "sp-std", + "sp-storage", + "static_assertions", "substrate-test-utils", ] @@ -4510,9 +4549,9 @@ version = "2.0.0-alpha.5" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote", + "quote 1.0.3", "sp-runtime", - "syn", + "syn 1.0.17", ] [[package]] @@ -4536,7 +4575,6 @@ dependencies = [ "frame-support", "frame-system", "parity-scale-codec", - "safe-mix", "sp-core", "sp-io", "sp-runtime", @@ -4625,6 +4663,7 @@ dependencies = [ name = "pallet-utility" version = "2.0.0-alpha.5" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", @@ -4673,6 +4712,24 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "parity-multiaddr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db35e222f783ef4e6661873f6c165c4eb7b65e0c408349818517d5705c2d7d3" +dependencies = [ + "arrayref", + "bs58", + "byteorder 1.3.4", + "data-encoding", + "multihash", + "percent-encoding 2.1.0", + "serde", + "static_assertions", + "unsigned-varint", + "url 2.1.1", +] + [[package]] name = "parity-multihash" version = "0.2.3" @@ -4690,9 +4747,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" +checksum = "329c8f7f4244ddb5c37c103641027a76c530e65e8e4b8240b29f81ea40508b17" dependencies = [ "arrayvec 0.5.1", "bitvec", @@ -4709,8 +4766,8 @@ checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -4744,7 +4801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ "proc-macro2", - "syn", + "syn 1.0.17", "synstructure", ] @@ -4815,9 +4872,9 @@ dependencies = [ [[package]] name = "paste" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +checksum = "092d791bf7847f70bbd49085489fba25fc2c193571752bff9e36e74e72403932" dependencies = [ "paste-impl", "proc-macro-hack", @@ -4825,14 +4882,14 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" +checksum = "406c23fb4c45cc6f68a9bbabb8ec7bd6f8cfcbd17e9e8f72c2460282f8325729" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -4895,8 +4952,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -4923,6 +4980,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "platforms" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" + [[package]] name = "plotters" version = "0.2.12" @@ -4930,7 +4993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3bb8da247d27ae212529352020f3e5ee16e83c0c258061d27b08ab92675eeb" dependencies = [ "js-sys", - "num-traits", + "num-traits 0.2.11", "wasm-bindgen", "web-sys", ] @@ -5003,68 +5066,78 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7959c6467d962050d639361f7703b2051c43036d03493c36f01d440fdd3138a" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" dependencies = [ "proc-macro-error-attr", "proc-macro2", - "quote", - "syn", - "version_check 0.9.1", + "quote 1.0.3", + "syn 1.0.17", + "version_check", ] [[package]] name = "proc-macro-error-attr" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4002d9f55991d5e019fb940a90e1a95eb80c24e77cb2462dd4dc869604d543a" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", "syn-mid", - "version_check 0.9.1", + "version_check", ] [[package]] name = "proc-macro-hack" -version = "0.5.11" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" [[package]] name = "proc-macro-nested" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" [[package]] name = "proc-macro2" -version = "1.0.9" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +dependencies = [ + "unicode-xid 0.2.0", +] + +[[package]] +name = "procfs" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +checksum = "fe50036aa1b71e553a4a0c48ab7baabf8aa8c7a5a61aae06bf38c2eab7430475" dependencies = [ - "unicode-xid", + "bitflags", + "byteorder 1.3.4", + "chrono", + "hex", + "lazy_static", + "libc", + "libflate", ] [[package]] name = "prometheus" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" +checksum = "b0575e258dab62268e7236d7307caa38848acbda7ec7ab87bd9093791e999d20" dependencies = [ "cfg-if", "fnv", "lazy_static", "protobuf", - "quick-error", "spin", + "thiserror", ] [[package]] @@ -5086,7 +5159,7 @@ dependencies = [ "bytes 0.5.4", "heck", "itertools", - "log 0.4.8", + "log", "multimap", "petgraph", "prost", @@ -5104,8 +5177,8 @@ dependencies = [ "anyhow", "itertools", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -5120,9 +5193,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.10.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a5325d019a4d837d3abde0a836920f959e33d350f77b5f1e289e061e774942" +checksum = "71964f34fd51cf04882d7ae3325fa0794d4cad66a03d0003f38d8ae4f63ba126" [[package]] name = "pwasm-utils" @@ -5131,7 +5204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7a12f176deee919f4ba55326ee17491c8b707d0987aed822682c821b660192" dependencies = [ "byteorder 1.3.4", - "log 0.4.8", + "log", "parity-wasm 0.41.0", ] @@ -5148,7 +5221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" dependencies = [ "env_logger 0.7.1", - "log 0.4.8", + "log", "rand 0.7.3", "rand_core 0.5.1", ] @@ -5164,6 +5237,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + [[package]] name = "quote" version = "1.0.3" @@ -5173,6 +5252,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac" + [[package]] name = "rand" version = "0.3.23" @@ -5436,9 +5521,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.4" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" dependencies = [ "aho-corasick", "memchr", @@ -5457,9 +5542,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.16" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" +checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" [[package]] name = "region" @@ -5484,9 +5569,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.11" +version = "0.16.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741ba1704ae21999c00942f9f5944f801e977f54302af346b596287599ad1862" +checksum = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c" dependencies = [ "cc", "lazy_static", @@ -5497,11 +5582,17 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" + [[package]] name = "rlp" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a44d5ae8afcb238af8b75640907edc6c931efcfab2c854e81ed35fa080f84cd" +checksum = "4a7d3f9bed94764eac15b8f14af59fac420c236adaff743b7bcc88e265cb4345" dependencies = [ "rustc-hex", ] @@ -5572,7 +5663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e" dependencies = [ "base64 0.10.1", - "log 0.4.8", + "log", "ring", "sct", "webpki", @@ -5585,7 +5676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" dependencies = [ "base64 0.11.0", - "log 0.4.8", + "log", "ring", "sct", "webpki", @@ -5610,8 +5701,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -5627,9 +5718,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" [[package]] name = "safe-mix" @@ -5640,12 +5731,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "safemem" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" - [[package]] name = "salsa20" version = "0.3.0" @@ -5685,7 +5770,7 @@ dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "libp2p", - "log 0.4.8", + "log", "parity-scale-codec", "prost", "prost-build", @@ -5711,7 +5796,7 @@ version = "0.8.0-alpha.5" dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-block-builder", @@ -5766,8 +5851,8 @@ version = "2.0.0-alpha.5" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -5784,8 +5869,9 @@ dependencies = [ "fdlimit", "futures 0.3.4", "lazy_static", - "log 0.4.8", + "log", "names", + "nix", "parity-util-mem", "regex", "rpassword", @@ -5802,6 +5888,7 @@ dependencies = [ "sp-panic-handler", "sp-runtime", "sp-state-machine", + "sp-utils", "structopt", "substrate-prometheus-endpoint", "tempfile", @@ -5821,7 +5908,7 @@ dependencies = [ "hex-literal", "kvdb", "kvdb-memorydb", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-block-builder", @@ -5840,6 +5927,7 @@ dependencies = [ "sp-state-machine", "sp-std", "sp-trie", + "sp-utils", "sp-version", "substrate-prometheus-endpoint", "substrate-test-runtime-client", @@ -5857,7 +5945,8 @@ dependencies = [ "hash-db", "hex-literal", "kvdb", - "log 0.4.8", + "lazy_static", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-executor", @@ -5876,6 +5965,7 @@ dependencies = [ "sp-test-primitives", "sp-transaction-pool", "sp-trie", + "sp-utils", "sp-version", ] @@ -5889,7 +5979,7 @@ dependencies = [ "kvdb-memorydb", "kvdb-rocksdb", "linked-hash-map", - "log 0.4.8", + "log", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", @@ -5919,7 +6009,7 @@ dependencies = [ "env_logger 0.7.1", "futures 0.3.4", "futures-timer 3.0.2", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-block-builder", @@ -5958,11 +6048,11 @@ dependencies = [ "fork-tree", "futures 0.3.4", "futures-timer 3.0.2", - "log 0.4.8", + "log", "merlin", "num-bigint", "num-rational", - "num-traits", + "num-traits 0.2.11", "parity-scale-codec", "parking_lot 0.10.0", "pdqselect", @@ -6047,7 +6137,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.8", + "log", "parking_lot 0.10.0", "sc-basic-authorship", "sc-client", @@ -6071,7 +6161,7 @@ version = "0.8.0-alpha.5" dependencies = [ "derive_more", "futures 0.3.4", - "log 0.4.8", + "log", "parity-scale-codec", "sc-client-api", "sp-api", @@ -6091,7 +6181,7 @@ version = "0.8.0-alpha.5" dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-client-api", @@ -6110,7 +6200,7 @@ dependencies = [ name = "sc-consensus-uncles" version = "0.8.0-alpha.5" dependencies = [ - "log 0.4.8", + "log", "sc-client-api", "sp-authorship", "sp-consensus", @@ -6128,7 +6218,7 @@ dependencies = [ "hex-literal", "lazy_static", "libsecp256k1", - "log 0.4.8", + "log", "parity-scale-codec", "parity-wasm 0.41.0", "parking_lot 0.10.0", @@ -6158,7 +6248,7 @@ name = "sc-executor-common" version = "0.8.0-alpha.5" dependencies = [ "derive_more", - "log 0.4.8", + "log", "parity-scale-codec", "sp-allocator", "sp-core", @@ -6172,7 +6262,7 @@ dependencies = [ name = "sc-executor-wasmi" version = "0.8.0-alpha.5" dependencies = [ - "log 0.4.8", + "log", "parity-scale-codec", "parity-wasm 0.41.0", "sc-executor-common", @@ -6188,7 +6278,7 @@ name = "sc-executor-wasmtime" version = "0.8.0-alpha.5" dependencies = [ "assert_matches", - "log 0.4.8", + "log", "parity-scale-codec", "parity-wasm 0.41.0", "sc-executor-common", @@ -6210,7 +6300,7 @@ dependencies = [ "fork-tree", "futures 0.3.4", "futures-timer 3.0.2", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "pin-project", @@ -6236,6 +6326,7 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-state-machine", + "sp-utils", "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", @@ -6248,7 +6339,7 @@ version = "0.8.0-alpha.5" dependencies = [ "ansi_term 0.12.1", "futures 0.3.4", - "log 0.4.8", + "log", "parity-util-mem", "sc-client-api", "sc-network", @@ -6294,7 +6385,7 @@ dependencies = [ "libp2p", "linked-hash-map", "linked_hash_set", - "log 0.4.8", + "log", "lru", "nohash-hasher", "parity-scale-codec", @@ -6321,6 +6412,7 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-test-primitives", + "sp-utils", "substrate-prometheus-endpoint", "substrate-test-runtime", "substrate-test-runtime-client", @@ -6339,10 +6431,11 @@ dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "libp2p", - "log 0.4.8", + "log", "lru", "sc-network", "sp-runtime", + "sp-utils", "substrate-test-runtime-client", "wasm-timer", ] @@ -6355,7 +6448,7 @@ dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "libp2p", - "log 0.4.8", + "log", "parking_lot 0.10.0", "rand 0.7.3", "sc-block-builder", @@ -6382,9 +6475,9 @@ dependencies = [ "fnv", "futures 0.3.4", "futures-timer 3.0.2", - "hyper 0.13.3", + "hyper 0.13.4", "hyper-rustls", - "log 0.4.8", + "log", "num_cpus", "parity-scale-codec", "parking_lot 0.10.0", @@ -6399,6 +6492,7 @@ dependencies = [ "sp-offchain", "sp-runtime", "sp-transaction-pool", + "sp-utils", "substrate-test-runtime-client", "threadpool", "tokio 0.2.13", @@ -6410,9 +6504,10 @@ version = "2.0.0-alpha.5" dependencies = [ "futures 0.3.4", "libp2p", - "log 0.4.8", + "log", "rand 0.7.3", "serde_json", + "sp-utils", "wasm-timer", ] @@ -6426,7 +6521,7 @@ dependencies = [ "hash-db", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "sc-block-builder", @@ -6448,6 +6543,7 @@ dependencies = [ "sp-session", "sp-state-machine", "sp-transaction-pool", + "sp-utils", "sp-version", "substrate-test-runtime-client", "tokio 0.1.22", @@ -6463,7 +6559,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "serde", @@ -6483,7 +6579,7 @@ dependencies = [ "jsonrpc-http-server", "jsonrpc-pubsub", "jsonrpc-ws-server", - "log 0.4.8", + "log", "serde", "serde_json", "sp-runtime", @@ -6513,11 +6609,13 @@ dependencies = [ "futures-diagnose", "futures-timer 3.0.2", "lazy_static", - "log 0.4.8", - "parity-multiaddr", + "log", + "netstat2", + "parity-multiaddr 0.7.3", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", + "procfs", "sc-chain-spec", "sc-client", "sc-client-api", @@ -6546,10 +6644,10 @@ dependencies = [ "sp-runtime", "sp-session", "sp-transaction-pool", + "sp-utils", "substrate-prometheus-endpoint", "substrate-test-runtime-client", "sysinfo", - "target_info", "tracing", "wasm-timer", ] @@ -6562,7 +6660,7 @@ dependencies = [ "fdlimit", "futures 0.1.29", "futures 0.3.4", - "log 0.4.8", + "log", "sc-client", "sc-network", "sc-service", @@ -6579,7 +6677,7 @@ name = "sc-state-db" version = "0.8.0-alpha.5" dependencies = [ "env_logger 0.7.1", - "log 0.4.8", + "log", "parity-scale-codec", "parity-util-mem", "parity-util-mem-derive", @@ -6596,7 +6694,7 @@ dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "libp2p", - "log 0.4.8", + "log", "parking_lot 0.10.0", "pin-project", "rand 0.7.3", @@ -6614,7 +6712,7 @@ name = "sc-tracing" version = "2.0.0-alpha.5" dependencies = [ "erased-serde", - "log 0.4.8", + "log", "parking_lot 0.10.0", "sc-telemetry", "serde", @@ -6633,7 +6731,7 @@ dependencies = [ "derive_more", "futures 0.3.4", "linked-hash-map", - "log 0.4.8", + "log", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", @@ -6642,6 +6740,7 @@ dependencies = [ "sp-core", "sp-runtime", "sp-transaction-pool", + "sp-utils", "substrate-test-runtime", "wasm-timer", ] @@ -6650,11 +6749,13 @@ dependencies = [ name = "sc-transaction-pool" version = "2.0.0-alpha.5" dependencies = [ + "assert_matches", "derive_more", "futures 0.3.4", "futures-diagnose", - "futures-timer 2.0.2", - "log 0.4.8", + "hex", + "intervalier", + "log", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", @@ -6666,6 +6767,7 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-transaction-pool", + "sp-utils", "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", "wasm-timer", @@ -6673,9 +6775,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507a9e6e8ffe0a4e0ebb9a10293e62fdf7657c06f1b8bb07a8fcf697d2abf295" +checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" dependencies = [ "lazy_static", "winapi 0.3.8", @@ -6727,8 +6829,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -6743,21 +6845,22 @@ dependencies = [ [[package]] name = "security-framework" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97bbedbe81904398b6ebb054b3e912f99d55807125790f3198ac990d98def5b0" +checksum = "572dfa3a0785509e7a44b5b4bebcf94d41ba34e9ed9eb9df722545c3b3c4144a" dependencies = [ "bitflags", "core-foundation", "core-foundation-sys", + "libc", "security-framework-sys", ] [[package]] name = "security-framework-sys" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06fd2f23e31ef68dd2328cc383bd493142e46107a3a0e24f7d734e3f3b80fe4c" +checksum = "8ddb15a5fec93b7021b8a9e96009c5d8d51c15673569f7c0f6b7204e5b7b404f" dependencies = [ "core-foundation-sys", "libc", @@ -6822,15 +6925,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] name = "serde_json" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +checksum = "78a7a12c167809363ec3bd7329fc0a3369056996de43c4b37ef3cd54a6ce4867" dependencies = [ "itoa", "ryu", @@ -6952,8 +7055,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -6999,9 +7102,9 @@ dependencies = [ "bytes 0.5.4", "flate2", "futures 0.3.4", - "http 0.2.0", + "http 0.2.1", "httparse", - "log 0.4.8", + "log", "rand 0.7.3", "sha1", "smallvec 1.2.0", @@ -7014,7 +7117,7 @@ name = "sp-allocator" version = "2.0.0-alpha.5" dependencies = [ "derive_more", - "log 0.4.8", + "log", "sp-core", "sp-std", "sp-wasm-interface", @@ -7042,8 +7145,8 @@ dependencies = [ "blake2-rfc", "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -7093,7 +7196,7 @@ version = "2.0.0-alpha.5" dependencies = [ "criterion 0.3.1", "integer-sqrt", - "num-traits", + "num-traits 0.2.11", "parity-scale-codec", "primitive-types", "rand 0.7.3", @@ -7102,6 +7205,17 @@ dependencies = [ "sp-std", ] +[[package]] +name = "sp-arithmetic-fuzzer" +version = "2.0.0-alpha.5" +dependencies = [ + "honggfuzz", + "num-bigint", + "num-traits 0.2.11", + "primitive-types", + "sp-arithmetic", +] + [[package]] name = "sp-authority-discovery" version = "2.0.0-alpha.5" @@ -7139,7 +7253,7 @@ name = "sp-blockchain" version = "2.0.0-alpha.5" dependencies = [ "derive_more", - "log 0.4.8", + "log", "lru", "parity-scale-codec", "parking_lot 0.10.0", @@ -7158,7 +7272,7 @@ dependencies = [ "futures-diagnose", "futures-timer 3.0.2", "libp2p", - "log 0.4.8", + "log", "parity-scale-codec", "parking_lot 0.10.0", "serde", @@ -7168,6 +7282,7 @@ dependencies = [ "sp-state-machine", "sp-std", "sp-test-primitives", + "sp-utils", "sp-version", ] @@ -7238,8 +7353,8 @@ dependencies = [ "impl-serde 0.3.0", "lazy_static", "libsecp256k1", - "log 0.4.8", - "num-traits", + "log", + "num-traits 0.2.11", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", @@ -7270,8 +7385,8 @@ name = "sp-debug-derive" version = "2.0.0-alpha.5" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -7321,7 +7436,7 @@ version = "2.0.0-alpha.5" dependencies = [ "hash-db", "libsecp256k1", - "log 0.4.8", + "log", "parity-scale-codec", "sp-core", "sp-externalities", @@ -7355,21 +7470,33 @@ name = "sp-panic-handler" version = "2.0.0-alpha.5" dependencies = [ "backtrace", - "log 0.4.8", + "log", ] [[package]] name = "sp-phragmen" version = "2.0.0-alpha.5" dependencies = [ + "parity-scale-codec", "rand 0.7.3", "serde", - "sp-io", + "sp-phragmen", + "sp-phragmen-compact", "sp-runtime", "sp-std", "substrate-test-utils", ] +[[package]] +name = "sp-phragmen-compact" +version = "2.0.0-dev" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote 1.0.3", + "syn 1.0.17", +] + [[package]] name = "sp-rpc" version = "2.0.0-alpha.5" @@ -7385,7 +7512,7 @@ version = "2.0.0-alpha.5" dependencies = [ "hash256-std-hasher", "impl-trait-for-tuples", - "log 0.4.8", + "log", "parity-scale-codec", "parity-util-mem", "paste", @@ -7426,8 +7553,8 @@ dependencies = [ "Inflector", "proc-macro-crate", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -7512,8 +7639,8 @@ version = "0.8.0-alpha.5" dependencies = [ "hash-db", "hex-literal", - "log 0.4.8", - "num-traits", + "log", + "num-traits 0.2.11", "parity-scale-codec", "parking_lot 0.10.0", "rand 0.7.3", @@ -7571,11 +7698,12 @@ version = "2.0.0-alpha.5" dependencies = [ "derive_more", "futures 0.3.4", - "log 0.4.8", + "log", "parity-scale-codec", "serde", "sp-api", "sp-runtime", + "sp-utils", ] [[package]] @@ -7596,6 +7724,16 @@ dependencies = [ "trie-standardmap", ] +[[package]] +name = "sp-utils" +version = "2.0.0-alpha.5" +dependencies = [ + "futures 0.3.4", + "futures-core", + "lazy_static", + "prometheus", +] + [[package]] name = "sp-version" version = "2.0.0-alpha.5" @@ -7679,9 +7817,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe43617218c0805c6eb37160119dc3c548110a67786da7218d1c6555212f073" +checksum = "c8faa2719539bbe9d77869bfb15d4ee769f99525e707931452c97b693b3f159d" dependencies = [ "clap", "lazy_static", @@ -7690,15 +7828,15 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e79c80e0f4efd86ca960218d4e056249be189ff1c42824dcd9a7f51a56f0bd" +checksum = "3f88b8e18c69496aad6f9ddf4630dd7d585bcaf765786cb415b9aec2fe5a0430" dependencies = [ "heck", "proc-macro-error", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -7718,8 +7856,8 @@ checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" dependencies = [ "heck", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -7777,7 +7915,7 @@ dependencies = [ "js-sys", "kvdb-web", "libp2p-wasm-ext", - "log 0.4.8", + "log", "rand 0.6.5", "rand 0.7.3", "sc-chain-spec", @@ -7791,6 +7929,9 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "2.0.0-alpha.5" +dependencies = [ + "platforms", +] [[package]] name = "substrate-frame-rpc-support" @@ -7818,7 +7959,7 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.8", + "log", "parity-scale-codec", "sc-client", "sc-transaction-pool", @@ -7838,8 +7979,8 @@ dependencies = [ "async-std", "derive_more", "futures-util", - "hyper 0.13.3", - "log 0.4.8", + "hyper 0.13.4", + "log", "prometheus", "tokio 0.2.13", ] @@ -7872,7 +8013,7 @@ dependencies = [ "frame-support", "frame-system", "frame-system-rpc-runtime-api", - "log 0.4.8", + "log", "memory-db", "pallet-babe", "pallet-timestamp", @@ -8043,13 +8184,24 @@ checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.16" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +dependencies = [ + "quote 0.3.15", + "synom", + "unicode-xid 0.0.4", +] + +[[package]] +name = "syn" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" dependencies = [ "proc-macro2", - "quote", - "unicode-xid", + "quote 1.0.3", + "unicode-xid 0.2.0", ] [[package]] @@ -8059,8 +8211,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +dependencies = [ + "unicode-xid 0.0.4", ] [[package]] @@ -8070,9 +8231,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ "proc-macro2", - "quote", - "syn", - "unicode-xid", + "quote 1.0.3", + "syn 1.0.17", + "unicode-xid 0.2.0", ] [[package]] @@ -8102,12 +8263,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" -[[package]] -name = "target_info" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" - [[package]] name = "tempfile" version = "3.1.0" @@ -8139,9 +8294,9 @@ checksum = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9" dependencies = [ "lazy_static", "proc-macro2", - "quote", - "syn", - "version_check 0.9.1", + "quote 1.0.3", + "syn 1.0.17", + "version_check", ] [[package]] @@ -8155,22 +8310,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db" +checksum = "f0570dc61221295909abdb95c739f2e74325e14293b2026b0a7e195091ec54ae" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4" +checksum = "227362df41d566be41a28f64401e07a043157c21c14b9785a0d8e256f940a8fd" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -8204,9 +8359,9 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2" dependencies = [ "failure", "hmac", @@ -8278,6 +8433,7 @@ checksum = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616" dependencies = [ "bytes 0.5.4", "fnv", + "futures-core", "iovec", "lazy_static", "libc", @@ -8364,7 +8520,7 @@ checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", "futures 0.1.29", - "log 0.4.8", + "log", ] [[package]] @@ -8374,8 +8530,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -8387,7 +8543,7 @@ dependencies = [ "crossbeam-utils", "futures 0.1.29", "lazy_static", - "log 0.4.8", + "log", "mio", "num_cpus", "parking_lot 0.9.0", @@ -8455,7 +8611,7 @@ dependencies = [ "crossbeam-utils", "futures 0.1.29", "lazy_static", - "log 0.4.8", + "log", "num_cpus", "slab", "tokio-executor 0.1.10", @@ -8473,17 +8629,6 @@ dependencies = [ "tokio-executor 0.1.10", ] -[[package]] -name = "tokio-tls" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "354b8cd83825b3c20217a9dc174d6a0c67441a2fae5c41bcb1ea6679f6ae0f7c" -dependencies = [ - "futures 0.1.29", - "native-tls", - "tokio-io", -] - [[package]] name = "tokio-udp" version = "0.1.6" @@ -8492,7 +8637,7 @@ checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" dependencies = [ "bytes 0.4.12", "futures 0.1.29", - "log 0.4.8", + "log", "mio", "tokio-codec", "tokio-io", @@ -8509,7 +8654,7 @@ dependencies = [ "futures 0.1.29", "iovec", "libc", - "log 0.4.8", + "log", "mio", "mio-uds", "tokio-codec", @@ -8519,14 +8664,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes 0.5.4", "futures-core", "futures-sink", - "log 0.4.8", + "log", "pin-project-lite", "tokio 0.2.13", ] @@ -8563,8 +8708,8 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fbad39da2f9af1cae3016339ad7f2c7a9e870f12e8fd04c4fd7ef35b30c0d2b" dependencies = [ - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", ] [[package]] @@ -8576,12 +8721,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "traitobject" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" - [[package]] name = "treeline" version = "0.1.0" @@ -8612,7 +8751,7 @@ checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" dependencies = [ "hash-db", "hashbrown", - "log 0.4.8", + "log", "rustc-hex", "smallvec 1.2.0", ] @@ -8673,15 +8812,9 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" dependencies = [ - "rand 0.6.5", + "rand 0.7.3", ] -[[package]] -name = "typeable" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" - [[package]] name = "typenum" version = "1.11.2" @@ -8700,22 +8833,13 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unicase" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" -dependencies = [ - "version_check 0.1.5", -] - [[package]] name = "unicase" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.1", + "version_check", ] [[package]] @@ -8748,6 +8872,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" + [[package]] name = "unicode-xid" version = "0.2.0" @@ -8812,22 +8942,6 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -[[package]] -name = "vergen" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" -dependencies = [ - "bitflags", - "chrono", -] - -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.1" @@ -8854,15 +8968,24 @@ dependencies = [ [[package]] name = "wabt-sys" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5d153dc96aad7dc13ab90835b892c69867948112d95299e522d370c4e13a08" +checksum = "23d7043ebb3e5d96fad7a8d3ca22ee9880748ff8c3e18092cfb2a49d3b8f9084" dependencies = [ "cc", "cmake", "glob 0.2.11", ] +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.1" @@ -8881,7 +9004,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ "futures 0.1.29", - "log 0.4.8", + "log", "try-lock", ] @@ -8891,7 +9014,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.8", + "log", "try-lock", ] @@ -8903,9 +9026,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3557c397ab5a8e347d434782bcd31fc1483d927a6826804cec05cc792ee2519d" +checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -8913,24 +9036,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0da9c9a19850d3af6df1cb9574970b566d617ecfaf36eb0b706b6f3ef9bd2f8" +checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.8", + "log", "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457414a91863c0ec00090dba537f88ab955d93ca6555862c29b6d860990b8a8a" +checksum = "7add542ea1ac7fdaa9dc25e031a6af33b7d63376292bd24140c637d00d1c312a" dependencies = [ "cfg-if", "js-sys", @@ -8940,32 +9063,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6fde1d36e75a714b5fe0cffbb78978f222ea6baebb726af13c78869fdb4205" +checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" dependencies = [ - "quote", + "quote 1.0.3", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bda4168030a6412ea8a047e27238cadf56f0e53516e1e83fec0a8b7c786f6d" +checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc9f36ad51f25b0219a3d4d13b90eb44cd075dff8b6280cca015775d7acaddd8" +checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" [[package]] name = "wasm-gc-api" @@ -8973,7 +9096,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c32691b6c7e6c14e7f8fd55361a9088b507aa49620fcd06c09b3a1082186b9" dependencies = [ - "log 0.4.8", + "log", "parity-wasm 0.32.0", "rustc-demangle", ] @@ -9004,7 +9127,7 @@ dependencies = [ "libc", "memory_units", "num-rational", - "num-traits", + "num-traits 0.2.11", "parity-wasm 0.41.0", "wasmi-validation", ] @@ -9057,7 +9180,7 @@ dependencies = [ "file-per-thread-logger", "indexmap", "libc", - "log 0.4.8", + "log", "more-asserts", "rayon", "serde", @@ -9087,27 +9210,27 @@ dependencies = [ [[package]] name = "wast" -version = "10.0.0" +version = "12.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4efb62ecebf5cc9dbf2954309a20d816289c6550c0597a138b9e811cefc05007" +checksum = "0615ba420811bcda39cf80e8a1bd75997aec09222bda35165920a07ef15cc695" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdea5e25273cc3a62f3ae3a1a4c7d7996625875b50c0b4475fee6698c2b069c" +checksum = "095f615fbfcae695e3a4cea7d9f02f70561c81274c0142f45a12bf1e154d08bd" dependencies = [ "wast", ] [[package]] name = "web-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721c6263e2c66fd44501cc5efbfa2b7dfa775d13e4ea38c46299646ed1f9c70a" +checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" dependencies = [ "js-sys", "wasm-bindgen", @@ -9141,47 +9264,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "websocket" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413b37840b9e27b340ce91b319ede10731de8c72f5bc4cb0206ec1ca4ce581d0" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.29", - "hyper 0.10.16", - "native-tls", - "rand 0.6.5", - "tokio-codec", - "tokio-io", - "tokio-reactor", - "tokio-tcp", - "tokio-tls", - "unicase 1.4.2", - "url 1.7.2", - "websocket-base", -] - -[[package]] -name = "websocket-base" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e3810f0d00c4dccb54c30a4eee815e703232819dec7b007db115791c42aa374" -dependencies = [ - "base64 0.10.1", - "bitflags", - "byteorder 1.3.4", - "bytes 0.4.12", - "futures 0.1.29", - "native-tls", - "rand 0.6.5", - "sha1", - "tokio-codec", - "tokio-io", - "tokio-tcp", - "tokio-tls", -] - [[package]] name = "which" version = "3.1.1" @@ -9221,9 +9303,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" dependencies = [ "winapi 0.3.8", ] @@ -9243,7 +9325,7 @@ dependencies = [ "byteorder 1.3.4", "bytes 0.4.12", "httparse", - "log 0.4.8", + "log", "mio", "mio-extras", "rand 0.7.3", @@ -9286,7 +9368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84300bb493cc878f3638b981c62b4632ec1a5c52daaa3036651e8c106d3b55ea" dependencies = [ "futures 0.3.4", - "log 0.4.8", + "log", "nohash-hasher", "parking_lot 0.10.0", "rand 0.7.3", @@ -9309,8 +9391,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ "proc-macro2", - "quote", - "syn", + "quote 1.0.3", + "syn 1.0.17", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index d86bab8abd5334d945fafadd345e32ec0b410423..035ae7bc37184f3de33c44ced4f0c3aa899768bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "bin/node-template/node", "bin/node-template/runtime", "bin/node-template/pallets/template", + "bin/node/bench", "bin/node/cli", "bin/node/executor", "bin/node/primitives", @@ -86,6 +87,7 @@ members = [ "frame/offences", "frame/randomness-collective-flip", "frame/recovery", + "frame/scheduler", "frame/scored-pool", "frame/session", "frame/session/benchmarking", @@ -130,6 +132,7 @@ members = [ "primitives/offchain", "primitives/panic-handler", "primitives/phragmen", + "primitives/phragmen/compact", "primitives/rpc", "primitives/runtime-interface", "primitives/runtime-interface/proc-macro", @@ -142,6 +145,7 @@ members = [ "primitives/api/proc-macro", "primitives/api/test", "primitives/arithmetic", + "primitives/arithmetic/fuzzer", "primitives/io", "primitives/runtime", "primitives/sandbox", @@ -153,6 +157,7 @@ members = [ "primitives/test-primitives", "primitives/transaction-pool", "primitives/trie", + "primitives/utils", "primitives/wasm-interface", "test-utils/client", "test-utils/runtime", @@ -170,3 +175,4 @@ members = [ [profile.release] # Substrate runtime requires unwinding. panic = "unwind" + diff --git a/bin/node-template/README.md b/bin/node-template/README.md index 4ae60478fc72b77f32664b4225c023c36ff5f1af..c1730d51e5c137e6a02ebee623f6a5fba25a2630 100644 --- a/bin/node-template/README.md +++ b/bin/node-template/README.md @@ -56,7 +56,7 @@ cargo run -- \ --chain=local \ --alice \ --node-key 0000000000000000000000000000000000000000000000000000000000000001 \ - --telemetry-url ws://telemetry.polkadot.io:1024 \ + --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ --validator ``` @@ -69,7 +69,7 @@ cargo run -- \ --chain=local \ --bob \ --port 30334 \ - --telemetry-url ws://telemetry.polkadot.io:1024 \ + --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ --validator ``` diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 201909bfd097d917573f23f449c5ff657217589b..a2ccc4fa95732b35da44cfa73e4165514ba2e4ea 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -2,6 +2,7 @@ name = "node-template" version = "2.0.0-alpha.5" authors = ["Anonymous"] +description = "Substrate Node template" edition = "2018" license = "Unlicense" build = "build.rs" @@ -12,7 +13,7 @@ repository = "https://github.com/paritytech/substrate/" name = "node-template" [dependencies] -futures = "0.3.1" +futures = "0.3.4" log = "0.4.8" structopt = "0.3.8" @@ -27,8 +28,8 @@ sc-network = { version = "0.8.0-alpha.5", path = "../../../client/network" } sc-consensus-aura = { version = "0.8.0-alpha.5", path = "../../../client/consensus/aura" } sp-consensus-aura = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/aura" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -grandpa = { version = "0.8.0-alpha.5", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -grandpa-primitives = { version = "2.0.0-alpha.5", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sc-finality-grandpa = { version = "0.8.0-alpha.5", path = "../../../client/finality-grandpa" } +sp-finality-grandpa = { version = "2.0.0-alpha.5", path = "../../../primitives/finality-grandpa" } sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } @@ -37,5 +38,7 @@ sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0. node-template-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } [build-dependencies] -vergen = "3.0.4" -build-script-utils = { version = "2.0.0-alpha.5", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } +substrate-build-script-utils = { version = "2.0.0-alpha.5", path = "../../../utils/build-script-utils" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node-template/node/build.rs b/bin/node-template/node/build.rs index 222cbb409285b40e7204cd609d444854dd4082aa..e3bfe3116bf28dba1872f7d0b64c2ee0c9c71c3c 100644 --- a/bin/node-template/node/build.rs +++ b/bin/node-template/node/build.rs @@ -1,9 +1,7 @@ -use vergen::{ConstantsFlags, generate_cargo_keys}; - -const ERROR_MSG: &str = "Failed to generate metadata files"; +use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; fn main() { - generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG); + generate_cargo_keys(); - build_script_utils::rerun_if_git_head_changed(); + rerun_if_git_head_changed(); } diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index f1a7e29d44cb44e297bbb65591b8ed8396a7c37d..8ed91698ebe8f796b06eaedf9ac46c63e302cd7e 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -4,7 +4,7 @@ use node_template_runtime::{ SudoConfig, SystemConfig, WASM_BINARY, Signature }; use sp_consensus_aura::sr25519::{AuthorityId as AuraId}; -use grandpa_primitives::{AuthorityId as GrandpaId}; +use sp_finality_grandpa::{AuthorityId as GrandpaId}; use sc_service; use sp_runtime::traits::{Verify, IdentifyAccount}; @@ -14,17 +14,6 @@ use sp_runtime::traits::{Verify, IdentifyAccount}; /// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type. pub type ChainSpec = sc_service::GenericChainSpec; -/// The chain specification option. This is expected to come in from the CLI and -/// is little more than one of a number of alternatives which can easily be converted -/// from a string (`--chain=...`) into a `ChainSpec`. -#[derive(Clone, Debug)] -pub enum Alternative { - /// Whatever the current runtime is, with just Alice as an auth. - Development, - /// Whatever the current runtime is, with simple Alice/Bob auths. - LocalTestnet, -} - /// Helper function to generate a crypto pair from seed pub fn get_from_seed(seed: &str) -> ::Public { TPublic::Pair::from_string(&format!("//{}", seed), None) @@ -42,80 +31,70 @@ pub fn get_account_id_from_seed(seed: &str) -> AccountId where } /// Helper function to generate an authority key for Aura -pub fn get_authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { +pub fn authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { ( get_from_seed::(s), get_from_seed::(s), ) } -impl Alternative { - /// Get an actual chain config from one of the alternatives. - pub(crate) fn load(self) -> Result { - Ok(match self { - Alternative::Development => ChainSpec::from_genesis( - "Development", - "dev", - || testnet_genesis( - vec![ - get_authority_keys_from_seed("Alice"), - ], - get_account_id_from_seed::("Alice"), - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - ], - true, - ), - vec![], - None, - None, - None, - None - ), - Alternative::LocalTestnet => ChainSpec::from_genesis( - "Local Testnet", - "local_testnet", - || testnet_genesis( - vec![ - get_authority_keys_from_seed("Alice"), - get_authority_keys_from_seed("Bob"), - ], - get_account_id_from_seed::("Alice"), - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Charlie"), - get_account_id_from_seed::("Dave"), - get_account_id_from_seed::("Eve"), - get_account_id_from_seed::("Ferdie"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - get_account_id_from_seed::("Charlie//stash"), - get_account_id_from_seed::("Dave//stash"), - get_account_id_from_seed::("Eve//stash"), - get_account_id_from_seed::("Ferdie//stash"), - ], - true, - ), - vec![], - None, - None, - None, - None - ), - }) - } +pub fn development_config() -> ChainSpec { + ChainSpec::from_genesis( + "Development", + "dev", + || testnet_genesis( + vec![ + authority_keys_from_seed("Alice"), + ], + get_account_id_from_seed::("Alice"), + vec![ + get_account_id_from_seed::("Alice"), + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + ], + true, + ), + vec![], + None, + None, + None, + None, + ) +} - pub(crate) fn from(s: &str) -> Option { - match s { - "dev" => Some(Alternative::Development), - "" | "local" => Some(Alternative::LocalTestnet), - _ => None, - } - } +pub fn local_testnet_config() -> ChainSpec { + ChainSpec::from_genesis( + "Local Testnet", + "local_testnet", + || testnet_genesis( + vec![ + authority_keys_from_seed("Alice"), + authority_keys_from_seed("Bob"), + ], + get_account_id_from_seed::("Alice"), + vec![ + get_account_id_from_seed::("Alice"), + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Charlie"), + get_account_id_from_seed::("Dave"), + get_account_id_from_seed::("Eve"), + get_account_id_from_seed::("Ferdie"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + get_account_id_from_seed::("Charlie//stash"), + get_account_id_from_seed::("Dave//stash"), + get_account_id_from_seed::("Eve//stash"), + get_account_id_from_seed::("Ferdie//stash"), + ], + true, + ), + vec![], + None, + None, + None, + None, + ) } fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, @@ -141,10 +120,3 @@ fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, }), } } - -pub fn load_spec(id: &str) -> Result, String> { - Ok(match Alternative::from(id) { - Some(spec) => Box::new(spec.load()?), - None => Box::new(ChainSpec::from_json_file(std::path::PathBuf::from(id))?), - }) -} diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 0f4c301dbff5b7defe0565bf71e85b21a6418867..75b88877aadfbe731dd16806e334948678fb24be 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -14,36 +14,64 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use sc_cli::VersionInfo; -use crate::service; use crate::chain_spec; use crate::cli::Cli; +use crate::service; +use sc_cli::SubstrateCli; +use sp_consensus_aura::sr25519::AuthorityPair as AuraPair; -/// Parse and run command line arguments -pub fn run(version: VersionInfo) -> sc_cli::Result<()> { - let opt = sc_cli::from_args::(&version); +impl SubstrateCli for Cli { + fn impl_name() -> &'static str { + "Substrate Node" + } + + fn impl_version() -> &'static str { + env!("SUBSTRATE_CLI_IMPL_VERSION") + } + + fn description() -> &'static str { + env!("CARGO_PKG_DESCRIPTION") + } + + fn author() -> &'static str { + env!("CARGO_PKG_AUTHORS") + } - let mut config = sc_service::Configuration::from_version(&version); + fn support_url() -> &'static str { + "support.anonymous.an" + } + + fn copyright_start_year() -> i32 { + 2017 + } + + fn executable_name() -> &'static str { + env!("CARGO_PKG_NAME") + } + + fn load_spec(&self, id: &str) -> Result, String> { + Ok(match id { + "dev" => Box::new(chain_spec::development_config()), + "" | "local" => Box::new(chain_spec::local_testnet_config()), + path => Box::new(chain_spec::ChainSpec::from_json_file( + std::path::PathBuf::from(path), + )?), + }) + } +} + +/// Parse and run command line arguments +pub fn run() -> sc_cli::Result<()> { + let cli = Cli::from_args(); - match opt.subcommand { + match &cli.subcommand { Some(subcommand) => { - subcommand.init(&version)?; - subcommand.update_config(&mut config, chain_spec::load_spec, &version)?; - subcommand.run( - config, - |config: _| Ok(new_full_start!(config).0), - ) - }, + let runner = cli.create_runner(subcommand)?; + runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0)) + } None => { - opt.run.init(&version)?; - opt.run.update_config(&mut config, chain_spec::load_spec, &version)?; - opt.run.run( - config, - service::new_light, - service::new_full, - &version, - ) - }, + let runner = cli.create_runner(&cli.run)?; + runner.run_node(service::new_light, service::new_full) + } } } diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 91b2c257e0cd733f7e87d00e02857101810a2fb0..369e6932a030811b542ae8de9f26e9324f22e069 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -8,16 +8,5 @@ mod cli; mod command; fn main() -> sc_cli::Result<()> { - let version = sc_cli::VersionInfo { - name: "Substrate Node", - commit: env!("VERGEN_SHA_SHORT"), - version: env!("CARGO_PKG_VERSION"), - executable_name: "node-template", - author: "Anonymous", - description: "Template Node", - support_url: "support.anonymous.an", - copyright_start_year: 2017, - }; - - command::run(version) + command::run() } diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 4b66f90fbb3732bbc07e59ced32a40717bd1b559..7c4a574f6be04c823e182671b1b7930bfdb5394d 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -10,7 +10,7 @@ use sp_inherents::InherentDataProviders; use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider}; +use sc_finality_grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider}; // Our native executor instance. native_executor_instance!( @@ -44,7 +44,7 @@ macro_rules! new_full_start { .ok_or_else(|| sc_service::Error::SelectChainRequired)?; let (grandpa_block_import, grandpa_link) = - grandpa::block_import(client.clone(), &(client.clone() as Arc<_>), select_chain)?; + sc_finality_grandpa::block_import(client.clone(), &(client.clone() as Arc<_>), select_chain)?; let aura_block_import = sc_consensus_aura::AuraBlockImport::<_, _, _, AuraPair>::new( grandpa_block_import.clone(), client.clone(), @@ -72,16 +72,11 @@ macro_rules! new_full_start { pub fn new_full(config: Configuration) -> Result { - let is_authority = config.roles.is_authority(); + let role = config.role.clone(); let force_authoring = config.force_authoring; - let name = config.name.clone(); + let name = config.network.node_name.clone(); let disable_grandpa = config.disable_grandpa; - // sentry nodes announce themselves as authorities to the network - // and should run the same protocols authorities do, but it should - // never actively participate in any consensus process. - let participates_in_consensus = is_authority && !config.sentry_mode; - let (builder, mut import_setup, inherent_data_providers) = new_full_start!(config); let (block_import, grandpa_link) = @@ -96,11 +91,9 @@ pub fn new_full(config: Configuration) })? .build()?; - if participates_in_consensus { - let proposer = sc_basic_authorship::ProposerFactory::new( - service.client(), - service.transaction_pool() - ); + if role.is_authority() { + let proposer = + sc_basic_authorship::ProposerFactory::new(service.client(), service.transaction_pool()); let client = service.client(); let select_chain = service.select_chain() @@ -129,20 +122,20 @@ pub fn new_full(config: Configuration) // if the node isn't actively participating in consensus then it doesn't // need a keystore, regardless of which protocol we use below. - let keystore = if participates_in_consensus { + let keystore = if role.is_authority() { Some(service.keystore()) } else { None }; - let grandpa_config = grandpa::Config { + let grandpa_config = sc_finality_grandpa::Config { // FIXME #1578 make this available through chainspec gossip_duration: Duration::from_millis(333), justification_period: 512, name: Some(name), observer_enabled: false, keystore, - is_authority, + is_authority: role.is_network_authority(), }; let enable_grandpa = !disable_grandpa; @@ -153,13 +146,13 @@ pub fn new_full(config: Configuration) // and vote data availability than the observer. The observer has not // been tested extensively yet and having most nodes in a network run it // could lead to finality stalls. - let grandpa_config = grandpa::GrandpaParams { + let grandpa_config = sc_finality_grandpa::GrandpaParams { config: grandpa_config, link: grandpa_link, network: service.network(), inherent_data_providers: inherent_data_providers.clone(), telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), + voting_rule: sc_finality_grandpa::VotingRulesBuilder::default().build(), prometheus_registry: service.prometheus_registry() }; @@ -167,10 +160,10 @@ pub fn new_full(config: Configuration) // if it fails we take down the service with it. service.spawn_essential_task( "grandpa-voter", - grandpa::run_grandpa_voter(grandpa_config)? + sc_finality_grandpa::run_grandpa_voter(grandpa_config)? ); } else { - grandpa::setup_disabled_grandpa( + sc_finality_grandpa::setup_disabled_grandpa( service.client(), &inherent_data_providers, service.network(), @@ -204,7 +197,7 @@ pub fn new_light(config: Configuration) let fetch_checker = fetcher .map(|fetcher| fetcher.checker().clone()) .ok_or_else(|| "Trying to start light import queue without active fetch checker")?; - let grandpa_block_import = grandpa::light_block_import( + let grandpa_block_import = sc_finality_grandpa::light_block_import( client.clone(), backend, &(client.clone() as Arc<_>), diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 285d05b560f96470e7f16ae581aae5fc734c2813..69fcd843526806c9fe9a0b6e684a292fcbfe778b 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -9,17 +9,15 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet template" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } -safe-mix = { default-features = false, version = '1.0.0' } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [dependencies.frame-support] default-features = false version = "2.0.0-alpha.5" path = "../../../../frame/support" -[dependencies.system] +[dependencies.frame-system] default-features = false -package = 'frame-system' version = "2.0.0-alpha.5" path = "../../../../frame/system" [dev-dependencies.sp-core] @@ -43,6 +41,8 @@ default = ['std'] std = [ 'codec/std', 'frame-support/std', - 'safe-mix/std', - 'system/std' + 'frame-system/std' ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index 892778adeb46b431cb46e87dd4e7ac4c1cea129b..a0daecfb72c9a14ffa7e7819400f0345c542b5b8 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -10,7 +10,7 @@ /// https://github.com/paritytech/substrate/blob/master/frame/example/src/lib.rs use frame_support::{decl_module, decl_storage, decl_event, decl_error, dispatch}; -use system::ensure_signed; +use frame_system::{self as system, ensure_signed}; #[cfg(test)] mod mock; @@ -75,6 +75,7 @@ decl_module! { /// Just a dummy entry point. /// function that can be called by the external world as an extrinsics call /// takes a parameter of the type `AccountId`, stores it, and emits an event + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn do_something(origin, something: u32) -> dispatch::DispatchResult { // Check it was signed and get the signer. See also: ensure_root and ensure_none let who = ensure_signed(origin)?; @@ -90,6 +91,7 @@ decl_module! { /// Another dummy entry point. /// takes no parameters, attempts to increment storage value, and possibly throws an error + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn cause_error(origin) -> dispatch::DispatchResult { // Check it was signed and get the signer. See also: ensure_root and ensure_none let _who = ensure_signed(origin)?; diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index 2ea81ffb456261122b234c6ae3c36f52eaa9c430..a93ac0359e3a2f2b8f4e0e558a0b254e33eb1c7e 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -6,6 +6,7 @@ use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; use sp_runtime::{ traits::{BlakeTwo256, IdentityLookup}, testing::Header, Perbill, }; +use frame_system as system; impl_outer_origin! { pub enum Origin for Test {} diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index c5f778e0556d8b94d206ec3fbfee7919052d1afe..e8653e6df70e82e503a54e79c638e9a796e41913 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } aura = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } balances = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } @@ -68,3 +68,6 @@ std = [ "transaction-payment/std", "template/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 0414759a5ad86f5c5be4adb1cc5031d4adf1b4e7..94f033fd8f58ee231f412e7a2cf6f7f53f77d49a 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -11,8 +11,8 @@ include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); use sp_std::prelude::*; use sp_core::OpaqueMetadata; use sp_runtime::{ - ApplyExtrinsicResult, transaction_validity::TransactionValidity, generic, create_runtime_str, - impl_opaque_keys, MultiSignature, + ApplyExtrinsicResult, generic, create_runtime_str, impl_opaque_keys, MultiSignature, + transaction_validity::{TransactionValidity, TransactionSource}, }; use sp_runtime::traits::{ BlakeTwo256, Block as BlockT, IdentityLookup, Verify, ConvertInto, IdentifyAccount @@ -318,8 +318,11 @@ impl_runtime_apis! { } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction(tx: ::Extrinsic) -> TransactionValidity { - Executive::validate_transaction(tx) + fn validate_transaction( + source: TransactionSource, + tx: ::Extrinsic, + ) -> TransactionValidity { + Executive::validate_transaction(source, tx) } } diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..22e7fe51d848e1f44ca0c34eddb57d39ca0d0c32 --- /dev/null +++ b/bin/node/bench/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "node-bench" +version = "0.8.0-alpha.5" +authors = ["Parity Technologies "] +description = "Substrate node integration benchmarks." +edition = "2018" +license = "GPL-3.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +log = "0.4.8" +node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } +node-testing = { version = "2.0.0-alpha.5", path = "../testing" } +sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" } +sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } +serde = "1.0.101" +serde_json = "1.0.41" +structopt = "0.3" \ No newline at end of file diff --git a/bin/node/bench/src/core.rs b/bin/node/bench/src/core.rs new file mode 100644 index 0000000000000000000000000000000000000000..a8164db75a53833855ad996cd1aceaa24e232123 --- /dev/null +++ b/bin/node/bench/src/core.rs @@ -0,0 +1,140 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{fmt, borrow::{Cow, ToOwned}}; +use serde::Serialize; + +pub struct Path(Vec); + +impl Path { + pub fn new(initial: &'static [&'static str]) -> Self { + Path(initial.iter().map(|x| x.to_string()).collect()) + } +} + +impl Path { + pub fn push(&mut self, item: &str) { + self.0.push(item.to_string()); + } + + pub fn full(&self) -> String { + self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val }) + } + + pub fn has(&self, path: &str) -> bool { + self.full().contains(path) + } +} + +pub trait BenchmarkDescription { + fn path(&self) -> Path; + + fn setup(self: Box) -> Box; + + fn name(&self) -> Cow<'static, str>; +} + +pub trait Benchmark { + fn run(&mut self) -> std::time::Duration; +} + +#[derive(Debug, Clone, Serialize)] +pub struct BenchmarkOutput { + name: String, + raw_average: u64, + average: u64, +} + +struct NsFormatter(u64); + +impl fmt::Display for NsFormatter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let v = self.0; + + if v < 100 { + return write!(f, "{} ns", v) + } + + if self.0 < 10_000 { + return write!(f, "{:.1} µs", v as f64 / 1000.0) + } + + if self.0 < 1_000_000 { + return write!(f, "{:.1} ms", v as f64 / 1_000_000.0) + } + + if self.0 < 100_000_000 { + return write!(f, "{} ms", v as f64 / 1_000_000.0) + } + + write!(f, "{:.2} s", v as f64 / 1_000_000_000.0) + } +} + +impl fmt::Display for BenchmarkOutput { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "({}: avg {}, w_avg {})", + self.name, + NsFormatter(self.raw_average), + NsFormatter(self.average), + ) + } +} + +pub fn run_benchmark(benchmark: Box) -> BenchmarkOutput { + let name = benchmark.name().to_owned(); + let mut benchmark = benchmark.setup(); + + let mut durations: Vec = vec![]; + for _ in 0..50 { + let duration = benchmark.run(); + durations.push(duration.as_nanos()); + } + + durations.sort(); + + let raw_average = (durations.iter().sum::() / (durations.len() as u128)) as u64; + let average = (durations.iter().skip(10).take(30).sum::() / 30) as u64; + + BenchmarkOutput { + name: name.into(), + raw_average, + average, + } +} + +macro_rules! matrix( + ( $var:ident in $over:expr => $tt:expr, $( $rest:tt )* ) => { + { + let mut res = Vec::>::new(); + for $var in $over.iter() { + res.push(Box::new($tt)); + } + res.extend(matrix!( $($rest)* )); + res + } + }; + ( $var:expr, $( $rest:tt )*) => { + { + let mut res = vec![Box::new($var) as Box]; + res.extend(matrix!( $($rest)* )); + res + } + }; + () => { vec![] } +); \ No newline at end of file diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs new file mode 100644 index 0000000000000000000000000000000000000000..20181bf4c7f7de816226ab76701d2c9339d56fa8 --- /dev/null +++ b/bin/node/bench/src/import.rs @@ -0,0 +1,136 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block import benchmark. +//! +//! This benchmark is expected to measure block import operation of +//! some more or less full block. +//! +//! As we also want to protect against cold-cache attacks, this +//! benchmark should not rely on any caching (except those that +//! DO NOT depend on user input). Thus block generation should be +//! based on randomized operation. +//! +//! This is supposed to be very simple benchmark and is not subject +//! to much configuring - just block full of randomized transactions. +//! It is not supposed to measure runtime modules weight correctness + +use std::borrow::Cow; + +use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes}; +use node_primitives::Block; +use sc_client_api::backend::Backend; +use sp_runtime::generic::BlockId; + +use crate::core::{self, Path}; + +#[derive(Clone, Copy, Debug)] +pub enum SizeType { Small, Medium, Large } + +impl SizeType { + fn transactions(&self) -> usize { + match self { + SizeType::Small => 10, + SizeType::Medium => 100, + SizeType::Large => 500, + } + } +} + +pub struct ImportBenchmarkDescription { + pub profile: Profile, + pub key_types: KeyTypes, + pub size: SizeType, +} + +pub struct ImportBenchmark { + profile: Profile, + database: BenchDb, + block: Block, +} + +impl core::BenchmarkDescription for ImportBenchmarkDescription { + fn path(&self) -> Path { + + let mut path = Path::new(&["node", "import"]); + + match self.profile { + Profile::Wasm => path.push("wasm"), + Profile::Native => path.push("native"), + } + + match self.key_types { + KeyTypes::Sr25519 => path.push("sr25519"), + KeyTypes::Ed25519 => path.push("ed25519"), + } + + match self.size { + SizeType::Small => path.push("small"), + SizeType::Medium => path.push("medium"), + SizeType::Large => path.push("large"), + } + + path + } + + fn setup(self: Box) -> Box { + let profile = self.profile; + let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types); + let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions())); + Box::new(ImportBenchmark { + database: bench_db, + block, + profile, + }) + } + + fn name(&self) -> Cow<'static, str> { + match self.profile { + Profile::Wasm => "Import benchmark (random transfers, wasm)".into(), + Profile::Native => "Import benchmark (random transfers, native)".into(), + } + } +} + +impl core::Benchmark for ImportBenchmark { + fn run(&mut self) -> std::time::Duration { + let mut context = self.database.create_context(self.profile); + + let _ = context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version") + .spec_version; + + let start = std::time::Instant::now(); + context.import_block(self.block.clone()); + let elapsed = start.elapsed(); + + log::info!( + target: "bench-logistics", + "imported block with {} tx, took: {:#?}", + self.block.extrinsics.len(), + elapsed, + ); + + log::info!( + target: "bench-logistics", + "usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ); + + elapsed + } +} \ No newline at end of file diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..8f045465262856fcb907f9aa5dfee68f7f05fc88 --- /dev/null +++ b/bin/node/bench/src/main.rs @@ -0,0 +1,95 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[macro_use] mod core; +mod import; + +use crate::core::run_benchmark; +use import::{ImportBenchmarkDescription, SizeType}; +use node_testing::bench::{Profile, KeyTypes}; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "node-bench", about = "Node integration benchmarks")] +struct Opt { + /// Show list of all available benchmarks. + /// + /// Will output ("name", "path"). Benchmarks can then be filtered by path. + #[structopt(short, long)] + list: bool, + + /// Machine readable json output. + /// + /// This also suppresses all regular output (except to stderr) + #[structopt(short, long)] + json: bool, + + /// Filter benchmarks. + /// + /// Run with `--list` for the hint of what to filter. + filter: Option, +} + +fn main() { + let opt = Opt::from_args(); + + if !opt.json { + sc_cli::init_logger(""); + } + + let benchmarks = matrix!( + profile in [Profile::Wasm, Profile::Native] => + ImportBenchmarkDescription { + profile: *profile, + key_types: KeyTypes::Sr25519, + size: SizeType::Medium, + }, + ImportBenchmarkDescription { + profile: Profile::Native, + key_types: KeyTypes::Ed25519, + size: SizeType::Medium, + }, + size in [SizeType::Small, SizeType::Large] => + ImportBenchmarkDescription { + profile: Profile::Native, + key_types: KeyTypes::Sr25519, + size: *size, + }, + ); + + if opt.list { + for benchmark in benchmarks.iter() { + log::info!("{}: {}", benchmark.name(), benchmark.path().full()) + } + return; + } + + let mut results = Vec::new(); + for benchmark in benchmarks { + if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) { + log::info!("Starting {}", benchmark.name()); + let result = run_benchmark(benchmark); + log::info!("{}", result); + + results.push(result); + } + } + + if opt.json { + let json_result: String = serde_json::to_string(&results).expect("Failed to construct json"); + println!("{}", json_result); + } +} \ No newline at end of file diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 9a286c5880085dcb1509f09db7a2ae190cef8df9..9842fd2ee49ac7650b441970393d3ba440723c64 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -2,7 +2,7 @@ name = "node-cli" version = "2.0.0-alpha.5" authors = ["Parity Technologies "] -description = "Substrate node implementation in Rust." +description = "Generic Substrate node implementation in Rust." build = "build.rs" edition = "2018" license = "GPL-3.0" @@ -16,7 +16,7 @@ repository = "https://github.com/paritytech/substrate/" wasm-opt = false [badges] -travis-ci = { repository = "paritytech/substrate", branch = "master" } +travis-ci = { repository = "paritytech/substrate" } maintenance = { status = "actively-developed" } is-it-maintained-issue-resolution = { repository = "paritytech/substrate" } is-it-maintained-open-issues = { repository = "paritytech/substrate" } @@ -31,7 +31,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } serde = { version = "1.0.102", features = ["derive"] } futures = { version = "0.3.1", features = ["compat"] } hex-literal = "0.2.1" @@ -82,6 +82,7 @@ pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/support" } pallet-im-online = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/im-online" } pallet-authority-discovery = { version = "2.0.0-alpha.5", path = "../../../frame/authority-discovery" } +pallet-staking = { version = "2.0.0-alpha.5", path = "../../../frame/staking" } # node-specific dependencies node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } @@ -100,23 +101,30 @@ wasm-bindgen = { version = "0.2.57", optional = true } wasm-bindgen-futures = { version = "0.4.7", optional = true } browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-alpha.5"} +[target.'cfg(target_arch="x86_64")'.dependencies] +node-executor = { version = "2.0.0-alpha.4", path = "../executor", features = [ "wasmtime" ] } +sc-cli = { version = "0.8.0-alpha.4", optional = true, path = "../../../client/cli", features = [ "wasmtime" ] } +sc-service = { version = "0.8.0-alpha.4", default-features = false, path = "../../../client/service", features = [ "wasmtime" ] } + [dev-dependencies] sc-keystore = { version = "2.0.0-alpha.5", path = "../../../client/keystore" } sc-consensus-babe = { version = "0.8.0-alpha.5", features = ["test-helpers"], path = "../../../client/consensus/babe" } sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../../client/consensus/epochs" } sc-service-test = { version = "2.0.0-dev", path = "../../../client/service/test" } -futures = "0.3.1" +futures = "0.3.4" tempfile = "3.1.0" -assert_cmd = "0.12" +assert_cmd = "1.0" nix = "0.17" serde_json = "1.0" +regex = "1" +platforms = "0.2.1" [build-dependencies] -build-script-utils = { version = "2.0.0-alpha.5", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } structopt = { version = "0.3.8", optional = true } node-transaction-factory = { version = "0.8.0-alpha.5", optional = true, path = "../transaction-factory" } node-inspect = { version = "0.8.0-alpha.5", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "2.0.0-alpha.5", optional = true, path = "../../../utils/frame/benchmarking-cli" } +substrate-build-script-utils = { version = "2.0.0-alpha.5", optional = true, path = "../../../utils/build-script-utils" } [build-dependencies.sc-cli] version = "0.8.0-alpha.5" @@ -124,12 +132,8 @@ package = "sc-cli" path = "../../../client/cli" optional = true -[build-dependencies.vergen] -version = "3.0.4" -optional = true - [features] -default = ["cli", "wasmtime"] +default = [ "cli" ] browser = [ "browser-utils", "wasm-bindgen", @@ -143,12 +147,9 @@ cli = [ "frame-benchmarking-cli", "sc-service/rocksdb", "structopt", - "vergen", -] -wasmtime = [ - "cli", - "node-executor/wasmtime", - "sc-cli/wasmtime", - "sc-service/wasmtime", + "substrate-build-script-utils", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index 8c4412667baceec56904864413178cbc88001497..cfad84a4cb5520f2542dc18da771d4cfa4a0a9ed 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -19,16 +19,5 @@ #![warn(missing_docs)] fn main() -> sc_cli::Result<()> { - let version = sc_cli::VersionInfo { - name: "Substrate Node", - commit: env!("VERGEN_SHA_SHORT"), - version: env!("CARGO_PKG_VERSION"), - executable_name: "substrate", - author: "Parity Technologies ", - description: "Generic substrate node", - support_url: "https://github.com/paritytech/substrate/issues/new", - copyright_start_year: 2017, - }; - - node_cli::run(std::env::args(), version) + node_cli::run() } diff --git a/bin/node/cli/build.rs b/bin/node/cli/build.rs index e824b59be64f3dedba3b738a6a3908b9e48b5091..12e0cab58ada5797c069e9a24970be6daa06de4f 100644 --- a/bin/node/cli/build.rs +++ b/bin/node/cli/build.rs @@ -24,14 +24,14 @@ mod cli { include!("src/cli.rs"); use std::{fs, env, path::Path}; - use sc_cli::{structopt::clap::Shell}; - use vergen::{ConstantsFlags, generate_cargo_keys}; + use sc_cli::structopt::clap::Shell; + use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; pub fn main() { build_shell_completion(); - generate_cargo_keys(ConstantsFlags::all()).expect("Failed to generate metadata files"); + generate_cargo_keys(); - build_script_utils::rerun_if_git_head_changed(); + rerun_if_git_head_changed(); } /// Build shell completion scripts for all known shells diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index e79ce273a32cc2ffde7ec1194fdbd81002e2fa25..6cd98dfe8d0d8ef0561406cb6a6b68fa6a3a0e88 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -41,11 +41,11 @@ async fn start_inner(chain_spec: String, log_level: String) -> Result ChainSpec { "staging_testnet", staging_testnet_config_genesis, boot_nodes, - Some(TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)])), + Some(TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) + .expect("Staging telemetry url is valid; qed")), None, None, Default::default(), @@ -181,7 +183,7 @@ pub fn get_account_id_from_seed(seed: &str) -> AccountId where } /// Helper function to generate stash, controller and session key from seed -pub fn get_authority_keys_from_seed(seed: &str) -> ( +pub fn authority_keys_from_seed(seed: &str) -> ( AccountId, AccountId, GrandpaId, @@ -201,7 +203,14 @@ pub fn get_authority_keys_from_seed(seed: &str) -> ( /// Helper function to create GenesisConfig for testing pub fn testnet_genesis( - initial_authorities: Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId, AuthorityDiscoveryId)>, + initial_authorities: Vec<( + AccountId, + AccountId, + GrandpaId, + BabeId, + ImOnlineId, + AuthorityDiscoveryId, + )>, root_key: AccountId, endowed_accounts: Option>, enable_println: bool, @@ -243,7 +252,12 @@ pub fn testnet_genesis( }), pallet_session: Some(SessionConfig { keys: initial_authorities.iter().map(|x| { - (x.0.clone(), x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone())) + (x.0.clone(), x.0.clone(), session_keys( + x.2.clone(), + x.3.clone(), + x.4.clone(), + x.5.clone(), + )) }).collect::>(), }), pallet_staking: Some(StakingConfig { @@ -257,13 +271,14 @@ pub fn testnet_genesis( .. Default::default() }), pallet_democracy: Some(DemocracyConfig::default()), - pallet_collective_Instance1: Some(CouncilConfig { + pallet_elections_phragmen: Some(ElectionsConfig { members: endowed_accounts.iter() .take((num_endowed_accounts + 1) / 2) .cloned() + .map(|member| (member, STASH)) .collect(), - phantom: Default::default(), }), + pallet_collective_Instance1: Some(CouncilConfig::default()), pallet_collective_Instance2: Some(TechnicalCommitteeConfig { members: endowed_accounts.iter() .take((num_endowed_accounts + 1) / 2) @@ -310,7 +325,7 @@ pub fn testnet_genesis( fn development_config_genesis() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), + authority_keys_from_seed("Alice"), ], get_account_id_from_seed::("Alice"), None, @@ -335,8 +350,8 @@ pub fn development_config() -> ChainSpec { fn local_testnet_genesis() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), - get_authority_keys_from_seed("Bob"), + authority_keys_from_seed("Alice"), + authority_keys_from_seed("Bob"), ], get_account_id_from_seed::("Alice"), None, @@ -368,7 +383,7 @@ pub(crate) mod tests { fn local_testnet_genesis_instant_single() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), + authority_keys_from_seed("Alice"), ], get_account_id_from_seed::("Alice"), None, diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index b6db9c3deb7e31f8f8da62d30b6971bf572b30d1..44b18fd716337c0857ebdf50f442001db12034c0 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::{SharedParams, ImportParams, RunCmd}; +use sc_cli::{ImportParams, RunCmd, SharedParams}; use structopt::StructOpt; /// An overarching CLI command definition. @@ -50,10 +50,7 @@ pub enum Subcommand { Inspect(node_inspect::cli::InspectCmd), /// The custom benchmark subcommmand benchmarking runtime pallets. - #[structopt( - name = "benchmark", - about = "Benchmark runtime pallets." - )] + #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), } @@ -62,11 +59,11 @@ pub enum Subcommand { #[derive(Debug, StructOpt, Clone)] pub struct FactoryCmd { /// Number of blocks to generate. - #[structopt(long="blocks", default_value = "1")] + #[structopt(long = "blocks", default_value = "1")] pub blocks: u32, /// Number of transactions to push per block. - #[structopt(long="transactions", default_value = "8")] + #[structopt(long = "transactions", default_value = "8")] pub transactions: u32, #[allow(missing_docs)] diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 21584f0898466dda3fca348bb40056b1436b43ac..37b77d3bb7475e31202d7592b4998e2c564084d5 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -14,102 +14,126 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::VersionInfo; -use sc_service::{Roles as ServiceRoles}; +use crate::{chain_spec, factory_impl::FactoryState, service, Cli, FactoryCmd, Subcommand}; +use node_executor::Executor; +use node_runtime::{Block, RuntimeApi}; use node_transaction_factory::RuntimeAdapter; -use crate::{Cli, service, ChainSpec, load_spec, Subcommand, factory_impl::FactoryState}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams, SubstrateCli}; +use sc_service::Configuration; + +impl SubstrateCli for Cli { + fn impl_name() -> &'static str { + "Substrate Node" + } + + fn impl_version() -> &'static str { + env!("SUBSTRATE_CLI_IMPL_VERSION") + } + + fn description() -> &'static str { + env!("CARGO_PKG_DESCRIPTION") + } + + fn author() -> &'static str { + env!("CARGO_PKG_AUTHORS") + } + + fn support_url() -> &'static str { + "https://github.com/paritytech/substrate/issues/new" + } + + fn copyright_start_year() -> i32 { + 2017 + } + + fn executable_name() -> &'static str { + "substrate" + } + + fn load_spec(&self, id: &str) -> std::result::Result, String> { + Ok(match id { + "dev" => Box::new(chain_spec::development_config()), + "local" => Box::new(chain_spec::local_testnet_config()), + "" | "fir" | "flaming-fir" => Box::new(chain_spec::flaming_fir_config()?), + "staging" => Box::new(chain_spec::staging_testnet_config()), + path => Box::new(chain_spec::ChainSpec::from_json_file( + std::path::PathBuf::from(path), + )?), + }) + } +} /// Parse command line arguments into service configuration. -pub fn run(args: I, version: VersionInfo) -> sc_cli::Result<()> -where - I: Iterator, - T: Into + Clone, -{ - let args: Vec<_> = args.collect(); - let opt = sc_cli::from_iter::(args.clone(), &version); +pub fn run() -> Result<()> { + sc_cli::reset_signal_pipe_handler()?; - let mut config = sc_service::Configuration::from_version(&version); + let cli = Cli::from_args(); - match opt.subcommand { + match &cli.subcommand { None => { - opt.run.init(&version)?; - opt.run.update_config(&mut config, load_spec, &version)?; - opt.run.run( - config, - service::new_light, - service::new_full, - &version, - ) - }, + let runner = cli.create_runner(&cli.run)?; + runner.run_node(service::new_light, service::new_full) + } Some(Subcommand::Inspect(cmd)) => { - cmd.init(&version)?; - cmd.update_config(&mut config, load_spec, &version)?; - - let client = sc_service::new_full_client::< - node_runtime::Block, node_runtime::RuntimeApi, node_executor::Executor, - >(&config)?; - let inspect = node_inspect::Inspector::::new(client); + let runner = cli.create_runner(cmd)?; - cmd.run(inspect) - }, + runner.sync_run(|config| cmd.run::(config)) + } Some(Subcommand::Benchmark(cmd)) => { - cmd.init(&version)?; - cmd.update_config(&mut config, load_spec, &version)?; - - cmd.run::(config) - }, - Some(Subcommand::Factory(cli_args)) => { - cli_args.shared_params.init(&version)?; - cli_args.shared_params.update_config(&mut config, load_spec, &version)?; - cli_args.import_params.update_config( - &mut config, - ServiceRoles::FULL, - cli_args.shared_params.dev, - )?; - - config.use_in_memory_keystore()?; - - match ChainSpec::from(config.expect_chain_spec().id()) { - Some(ref c) if c == &ChainSpec::Development || c == &ChainSpec::LocalTestnet => {}, - _ => return Err( - "Factory is only supported for development and local testnet.".into() - ), - } + let runner = cli.create_runner(cmd)?; - // Setup tracing. - if let Some(tracing_targets) = cli_args.import_params.tracing_targets.as_ref() { - let subscriber = sc_tracing::ProfilingSubscriber::new( - cli_args.import_params.tracing_receiver.into(), tracing_targets - ); - if let Err(e) = tracing::subscriber::set_global_default(subscriber) { - return Err( - format!("Unable to set global default subscriber {}", e).into() - ); - } - } + runner.sync_run(|config| cmd.run::(config)) + } + Some(Subcommand::Factory(cmd)) => { + let runner = cli.create_runner(cmd)?; - let factory_state = FactoryState::new( - cli_args.blocks, - cli_args.transactions, - ); + runner.sync_run(|config| cmd.run(config)) + } + Some(Subcommand::Base(subcommand)) => { + let runner = cli.create_runner(subcommand)?; + + runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0)) + } + } +} - let service_builder = new_full_start!(config).0; - node_transaction_factory::factory( - factory_state, - service_builder.client(), - service_builder.select_chain() - .expect("The select_chain is always initialized by new_full_start!; QED") - ).map_err(|e| format!("Error in transaction factory: {}", e))?; +impl CliConfiguration for FactoryCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) - }, - Some(Subcommand::Base(subcommand)) => { - subcommand.init(&version)?; - subcommand.update_config(&mut config, load_spec, &version)?; - subcommand.run( - config, - |config: sc_service::Configuration| Ok(new_full_start!(config).0), - ) - }, + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} + +impl FactoryCmd { + fn run(&self, config: Configuration) -> Result<()> { + match config.chain_spec.id() { + "dev" | "local" => {} + _ => return Err("Factory is only supported for development and local testnet.".into()), + } + + // Setup tracing. + if let Some(tracing_targets) = self.import_params.tracing_targets.as_ref() { + let subscriber = sc_tracing::ProfilingSubscriber::new( + self.import_params.tracing_receiver.into(), + tracing_targets, + ); + if let Err(e) = tracing::subscriber::set_global_default(subscriber) { + return Err(format!("Unable to set global default subscriber {}", e).into()); + } + } + + let factory_state = FactoryState::new(self.blocks, self.transactions); + + let service_builder = new_full_start!(config).0; + node_transaction_factory::factory( + factory_state, + service_builder.client(), + service_builder + .select_chain() + .expect("The select_chain is always initialized by new_full_start!; qed"), + ) } } diff --git a/bin/node/cli/src/factory_impl.rs b/bin/node/cli/src/factory_impl.rs index 1d1eabe29cbfb12ca43663be00b0969e852a30d9..cd7e3022e0b9a66e510707245118dbc123edee0d 100644 --- a/bin/node/cli/src/factory_impl.rs +++ b/bin/node/cli/src/factory_impl.rs @@ -58,6 +58,7 @@ impl FactoryState { frame_system::CheckWeight::new(), pallet_transaction_payment::ChargeTransactionPayment::from(0), Default::default(), + Default::default(), ) } } @@ -122,7 +123,7 @@ impl RuntimeAdapter for FactoryState { (*amount).into() ) ) - }, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), (), ())) + }, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), (), (), ())) } fn inherent_extrinsics(&self) -> InherentData { diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 6b3644856c6b027172ad62b132c65fcb92d05146..1e2c790bfa7d21a8857ba1bf2f90a3a419701585 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -47,45 +47,3 @@ pub use browser::*; pub use cli::*; #[cfg(feature = "cli")] pub use command::*; - -/// The chain specification option. -#[derive(Clone, Debug, PartialEq)] -pub enum ChainSpec { - /// Whatever the current runtime is, with just Alice as an auth. - Development, - /// Whatever the current runtime is, with simple Alice/Bob auths. - LocalTestnet, - /// The Flaming Fir testnet. - FlamingFir, - /// Whatever the current runtime is with the "global testnet" defaults. - StagingTestnet, -} - -/// Get a chain config from a spec setting. -impl ChainSpec { - pub(crate) fn load(self) -> Result { - Ok(match self { - ChainSpec::FlamingFir => chain_spec::flaming_fir_config()?, - ChainSpec::Development => chain_spec::development_config(), - ChainSpec::LocalTestnet => chain_spec::local_testnet_config(), - ChainSpec::StagingTestnet => chain_spec::staging_testnet_config(), - }) - } - - pub(crate) fn from(s: &str) -> Option { - match s { - "dev" => Some(ChainSpec::Development), - "local" => Some(ChainSpec::LocalTestnet), - "" | "fir" | "flaming-fir" => Some(ChainSpec::FlamingFir), - "staging" => Some(ChainSpec::StagingTestnet), - _ => None, - } - } -} - -fn load_spec(id: &str) -> Result, String> { - Ok(match ChainSpec::from(id) { - Some(spec) => Box::new(spec.load()?), - None => Box::new(chain_spec::ChainSpec::from_json_file(std::path::PathBuf::from(id))?), - }) -} diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index d81ec5f031ff45e3c6cec049e5adc622443fabfd..92d62364a27f9d7515012c1433a382ae4eb9f03d 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -88,7 +88,7 @@ macro_rules! new_full_start { import_setup = Some((block_import, grandpa_link, babe_link)); Ok(import_queue) })? - .with_rpc_extensions(|builder| -> Result { + .with_rpc_extensions(|builder| -> std::result::Result { let babe_link = import_setup.as_ref().map(|s| &s.2) .expect("BabeLink is present for full services or set up failed; qed."); let deps = node_rpc::FullDeps { @@ -120,24 +120,17 @@ macro_rules! new_full { use sc_client_api::ExecutorProvider; let ( - is_authority, + role, force_authoring, name, disable_grandpa, - sentry_nodes, ) = ( - $config.roles.is_authority(), + $config.role.clone(), $config.force_authoring, - $config.name.clone(), + $config.network.node_name.clone(), $config.disable_grandpa, - $config.network.sentry_nodes.clone(), ); - // sentry nodes announce themselves as authorities to the network - // and should run the same protocols authorities do, but it should - // never actively participate in any consensus process. - let participates_in_consensus = is_authority && !$config.sentry_mode; - let (builder, mut import_setup, inherent_data_providers) = new_full_start!($config); let service = builder @@ -153,7 +146,7 @@ macro_rules! new_full { ($with_startup_data)(&block_import, &babe_link); - if participates_in_consensus { + if let sc_service::config::Role::Authority { sentry_nodes } = &role { let proposer = sc_basic_authorship::ProposerFactory::new( service.client(), service.transaction_pool() @@ -190,7 +183,7 @@ macro_rules! new_full { let authority_discovery = sc_authority_discovery::AuthorityDiscovery::new( service.client(), network, - sentry_nodes, + sentry_nodes.clone(), service.keystore(), dht_event_stream, service.prometheus_registry(), @@ -201,7 +194,7 @@ macro_rules! new_full { // if the node isn't actively participating in consensus then it doesn't // need a keystore, regardless of which protocol we use below. - let keystore = if participates_in_consensus { + let keystore = if role.is_authority() { Some(service.keystore()) } else { None @@ -214,7 +207,7 @@ macro_rules! new_full { name: Some(name), observer_enabled: false, keystore, - is_authority, + is_authority: role.is_network_authority(), }; let enable_grandpa = !disable_grandpa; @@ -624,11 +617,12 @@ mod tests { check_weight, payment, Default::default(), + Default::default(), ); let raw_payload = SignedPayload::from_raw( function, extra, - (version, genesis_hash, genesis_hash, (), (), (), ()) + (version, genesis_hash, genesis_hash, (), (), (), (), ()) ); let signature = raw_payload.using_encoded(|payload| { signer.sign(payload) diff --git a/bin/node/cli/tests/version.rs b/bin/node/cli/tests/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..5555efd3854d4e08ecae60dd0c57ecf05b22e040 --- /dev/null +++ b/bin/node/cli/tests/version.rs @@ -0,0 +1,83 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use assert_cmd::cargo::cargo_bin; +use platforms::*; +use regex::Regex; +use std::process::Command; + +fn expected_regex() -> Regex { + Regex::new(r"^substrate (\d+\.\d+\.\d+(?:-.+?)?)-([a-f\d]+)-(.+?)-(.+?)(?:-(.+))?$").unwrap() +} + +#[test] +fn version_is_full() { + let expected = expected_regex(); + let output = Command::new(cargo_bin("substrate")) + .args(&["--version"]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "command returned with non-success exit code" + ); + + let output = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + let captures = expected + .captures(output.as_str()) + .expect("could not parse version in output"); + + assert_eq!(&captures[1], env!("CARGO_PKG_VERSION")); + assert_eq!(&captures[3], TARGET_ARCH.as_str()); + assert_eq!(&captures[4], TARGET_OS.as_str()); + assert_eq!( + captures.get(5).map(|x| x.as_str()), + TARGET_ENV.map(|x| x.as_str()) + ); +} + +#[test] +fn test_regex_matches_properly() { + let expected = expected_regex(); + + let captures = expected + .captures("substrate 2.0.0-da487d19d-x86_64-linux-gnu") + .unwrap(); + assert_eq!(&captures[1], "2.0.0"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), Some("gnu")); + + let captures = expected + .captures("substrate 2.0.0-alpha.5-da487d19d-x86_64-linux-gnu") + .unwrap(); + assert_eq!(&captures[1], "2.0.0-alpha.5"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), Some("gnu")); + + let captures = expected + .captures("substrate 2.0.0-alpha.5-da487d19d-x86_64-linux") + .unwrap(); + assert_eq!(&captures[1], "2.0.0-alpha.5"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), None); +} diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 02c6a141e9e4beaef558f7dde8ae32bf939886e6..2f1060a99884505ba94f1692f493bf8cdb01f43a 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -9,7 +9,7 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor" } @@ -52,3 +52,6 @@ stress-test = [] [[bench]] name = "bench" harness = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 1ee0a17c81120ca32a7a9af2d704ce97699ee454..fccf4a62cc21dbe81f3fa72f6ab2fde8ae6fceda 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -338,7 +338,7 @@ fn full_native_block_import_works() { EventRecord { phase: Phase::ApplyExtrinsic(0), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 10000, class: DispatchClass::Operational, pays_fee: true } + DispatchInfo { weight: 10000, class: DispatchClass::Mandatory, pays_fee: true } )), topics: vec![], }, @@ -391,7 +391,7 @@ fn full_native_block_import_works() { EventRecord { phase: Phase::ApplyExtrinsic(0), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 10000, class: DispatchClass::Operational, pays_fee: true } + DispatchInfo { weight: 10000, class: DispatchClass::Mandatory, pays_fee: true } )), topics: vec![], }, diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 1a92aeca6ba77b61597f3f0d19317f67adca2b03..536cf486e38ae28e0df13fadd65ac9b5c20e99a0 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . use node_runtime::{ - Call, Executive, Indices, Runtime, SubmitTransaction, UncheckedExtrinsic, + Call, Executive, Indices, Runtime, TransactionSubmitterOf, UncheckedExtrinsic, }; use sp_application_crypto::AppKey; use sp_core::testing::KeyStore; @@ -31,6 +31,8 @@ use codec::Decode; pub mod common; use self::common::*; +type SubmitTransaction = TransactionSubmitterOf; + #[test] fn should_submit_unsigned_transaction() { let mut t = new_test_ext(COMPACT_CODE, false); @@ -138,7 +140,7 @@ fn should_submit_signed_twice_from_the_same_account() { fn submitted_transaction_should_be_valid() { use codec::Encode; use frame_support::storage::StorageMap; - use sp_runtime::transaction_validity::ValidTransaction; + use sp_runtime::transaction_validity::{ValidTransaction, TransactionSource}; use sp_runtime::traits::StaticLookup; let mut t = new_test_ext(COMPACT_CODE, false); @@ -163,6 +165,7 @@ fn submitted_transaction_should_be_valid() { let tx0 = state.read().transactions[0].clone(); let mut t = new_test_ext(COMPACT_CODE, false); t.execute_with(|| { + let source = TransactionSource::External; let extrinsic = UncheckedExtrinsic::decode(&mut &*tx0).unwrap(); // add balance to the account let author = extrinsic.signature.clone().unwrap().0; @@ -172,15 +175,14 @@ fn submitted_transaction_should_be_valid() { >::insert(&address, account); // check validity - let res = Executive::validate_transaction(extrinsic); + let res = Executive::validate_transaction(source, extrinsic); assert_eq!(res.unwrap(), ValidTransaction { priority: 2_411_002_000_000, requires: vec![], provides: vec![(address, 0).encode()], - longevity: 127, + longevity: 128, propagate: true, }); }); } - diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml index 36ba0c7205be5d558096566af6c95a81b6e2dd32..9e94fe74d68469d9570fe76292d004e9106ef049 100644 --- a/bin/node/inspect/Cargo.toml +++ b/bin/node/inspect/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99" log = "0.4.8" sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } @@ -18,3 +18,6 @@ sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockch sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } structopt = "0.3.8" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs index 16bfda2bd927a8890d3416cfe8b75e96d216b117..2212907f763138feef1607b2236bdbbf7e0867a3 100644 --- a/bin/node/inspect/src/command.rs +++ b/bin/node/inspect/src/command.rs @@ -16,186 +16,48 @@ //! Command ran by the CLI -use std::{ - fmt::Debug, - str::FromStr, -}; - use crate::cli::{InspectCmd, InspectSubCmd}; -use crate::{Inspector, PrettyPrinter}; +use crate::Inspector; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_service::{new_full_client, Configuration, NativeExecutionDispatch}; +use sp_runtime::traits::Block; +use std::str::FromStr; impl InspectCmd { - /// Initialize - pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { - self.shared_params.init(version) - } - - /// Parse CLI arguments and initialize given config. - pub fn update_config( - &self, - mut config: &mut sc_service::config::Configuration, - spec_factory: impl FnOnce(&str) -> Result, String>, - version: &sc_cli::VersionInfo, - ) -> sc_cli::Result<()> { - self.shared_params.update_config(config, spec_factory, version)?; - - // make sure to configure keystore - config.use_in_memory_keystore()?; - - // and all import params (especially pruning that has to match db meta) - self.import_params.update_config( - &mut config, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - - Ok(()) - } - /// Run the inspect command, passing the inspector. - pub fn run( - self, - inspect: Inspector, - ) -> sc_cli::Result<()> where - B: sp_runtime::traits::Block, + pub fn run(&self, config: Configuration) -> Result<()> + where + B: Block, B::Hash: FromStr, - P: PrettyPrinter, + RA: Send + Sync + 'static, + EX: NativeExecutionDispatch + 'static, { - match self.command { + let client = new_full_client::(&config)?; + let inspect = Inspector::::new(client); + + match &self.command { InspectSubCmd::Block { input } => { let input = input.parse()?; - let res = inspect.block(input) - .map_err(|e| format!("{}", e))?; + let res = inspect.block(input).map_err(|e| format!("{}", e))?; println!("{}", res); Ok(()) - }, + } InspectSubCmd::Extrinsic { input } => { let input = input.parse()?; - let res = inspect.extrinsic(input) - .map_err(|e| format!("{}", e))?; + let res = inspect.extrinsic(input).map_err(|e| format!("{}", e))?; println!("{}", res); Ok(()) - }, - } - } -} - -/// A block to retrieve. -#[derive(Debug, Clone, PartialEq)] -pub enum BlockAddress { - /// Get block by hash. - Hash(Hash), - /// Get block by number. - Number(Number), - /// Raw SCALE-encoded bytes. - Bytes(Vec), -} - -impl FromStr for BlockAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // try to parse hash first - if let Ok(hash) = s.parse() { - return Ok(Self::Hash(hash)) - } - - // then number - if let Ok(number) = s.parse() { - return Ok(Self::Number(number)) + } } - - // then assume it's bytes (hex-encoded) - sp_core::bytes::from_hex(s) - .map(Self::Bytes) - .map_err(|e| format!( - "Given string does not look like hash or number. It could not be parsed as bytes either: {}", - e - )) - } -} - -/// An extrinsic address to decode and print out. -#[derive(Debug, Clone, PartialEq)] -pub enum ExtrinsicAddress { - /// Extrinsic as part of existing block. - Block(BlockAddress, usize), - /// Raw SCALE-encoded extrinsic bytes. - Bytes(Vec), -} - -impl FromStr for ExtrinsicAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // first try raw bytes - if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { - return Ok(bytes) - } - - // split by a bunch of different characters - let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); - let block = it.next() - .expect("First element of split iterator is never empty; qed") - .parse()?; - - let index = it.next() - .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? - .parse() - .map_err(|e| format!("Invalid index format: {}", e))?; - - Ok(Self::Block(block, index)) } } -#[cfg(test)] -mod tests { - use super::*; - use sp_core::hash::H160 as Hash; - - #[test] - fn should_parse_block_strings() { - type BlockAddress = super::BlockAddress; - - let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); - let b1 = BlockAddress::from_str("1234"); - let b2 = BlockAddress::from_str("0"); - let b3 = BlockAddress::from_str("0x0012345f"); - - - assert_eq!(b0, Ok(BlockAddress::Hash( - "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() - ))); - assert_eq!(b1, Ok(BlockAddress::Number(1234))); - assert_eq!(b2, Ok(BlockAddress::Number(0))); - assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); +impl CliConfiguration for InspectCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params } - #[test] - fn should_parse_extrinsic_address() { - type BlockAddress = super::BlockAddress; - type ExtrinsicAddress = super::ExtrinsicAddress; - - let e0 = ExtrinsicAddress::from_str("1234"); - let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); - let b1 = ExtrinsicAddress::from_str("1234:0"); - let b2 = ExtrinsicAddress::from_str("0 0"); - let b3 = ExtrinsicAddress::from_str("0x0012345f"); - - - assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); - assert_eq!(b0, Ok(ExtrinsicAddress::Block( - BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), - 5 - ))); - assert_eq!(b1, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(1234), - 0 - ))); - assert_eq!(b2, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(0), - 0 - ))); - assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs index c82682d6021dbbac42f51c0367343d1dec8dcb65..b8101d98a31ce717b4a57f9ee939abe2444253a9 100644 --- a/bin/node/inspect/src/lib.rs +++ b/bin/node/inspect/src/lib.rs @@ -27,7 +27,9 @@ pub mod command; use std::{ fmt, - marker::PhantomData + fmt::Debug, + marker::PhantomData, + str::FromStr, }; use codec::{Encode, Decode}; use sc_client_api::BlockBackend; @@ -38,8 +40,6 @@ use sp_runtime::{ traits::{Block, HashFor, NumberFor, Hash} }; -use command::{BlockAddress, ExtrinsicAddress}; - /// A helper type for a generic block input. pub type BlockAddressFor = BlockAddress< as Hash>::Output, @@ -205,3 +205,123 @@ impl> Inspector Ok(format!("{}", ExtrinsicPrinter(ext, &self.printer))) } } + +/// A block to retrieve. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockAddress { + /// Get block by hash. + Hash(Hash), + /// Get block by number. + Number(Number), + /// Raw SCALE-encoded bytes. + Bytes(Vec), +} + +impl FromStr for BlockAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // try to parse hash first + if let Ok(hash) = s.parse() { + return Ok(Self::Hash(hash)) + } + + // then number + if let Ok(number) = s.parse() { + return Ok(Self::Number(number)) + } + + // then assume it's bytes (hex-encoded) + sp_core::bytes::from_hex(s) + .map(Self::Bytes) + .map_err(|e| format!( + "Given string does not look like hash or number. It could not be parsed as bytes either: {}", + e + )) + } +} + +/// An extrinsic address to decode and print out. +#[derive(Debug, Clone, PartialEq)] +pub enum ExtrinsicAddress { + /// Extrinsic as part of existing block. + Block(BlockAddress, usize), + /// Raw SCALE-encoded extrinsic bytes. + Bytes(Vec), +} + +impl FromStr for ExtrinsicAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // first try raw bytes + if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { + return Ok(bytes) + } + + // split by a bunch of different characters + let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); + let block = it.next() + .expect("First element of split iterator is never empty; qed") + .parse()?; + + let index = it.next() + .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? + .parse() + .map_err(|e| format!("Invalid index format: {}", e))?; + + Ok(Self::Block(block, index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::hash::H160 as Hash; + + #[test] + fn should_parse_block_strings() { + type BlockAddress = super::BlockAddress; + + let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); + let b1 = BlockAddress::from_str("1234"); + let b2 = BlockAddress::from_str("0"); + let b3 = BlockAddress::from_str("0x0012345f"); + + + assert_eq!(b0, Ok(BlockAddress::Hash( + "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() + ))); + assert_eq!(b1, Ok(BlockAddress::Number(1234))); + assert_eq!(b2, Ok(BlockAddress::Number(0))); + assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } + + #[test] + fn should_parse_extrinsic_address() { + type BlockAddress = super::BlockAddress; + type ExtrinsicAddress = super::ExtrinsicAddress; + + let e0 = ExtrinsicAddress::from_str("1234"); + let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); + let b1 = ExtrinsicAddress::from_str("1234:0"); + let b2 = ExtrinsicAddress::from_str("0 0"); + let b3 = ExtrinsicAddress::from_str("0x0012345f"); + + + assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); + assert_eq!(b0, Ok(ExtrinsicAddress::Block( + BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), + 5 + ))); + assert_eq!(b1, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(1234), + 0 + ))); + assert_eq!(b2, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(0), + 0 + ))); + assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } +} diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index d4d6d990923a495cec8bc0830f6944d9877af4dd..81c5009f394d57332d5c9ef65ed8c6bb2cb6f120 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -21,3 +21,6 @@ std = [ "sp-core/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml index ec6839b71c1147d23a07483fa3e3bc0df7c27d8b..df095bc5bb10d2574a01790f3ed32ea9c8585ebe 100644 --- a/bin/node/rpc-client/Cargo.toml +++ b/bin/node/rpc-client/Cargo.toml @@ -11,7 +11,10 @@ repository = "https://github.com/paritytech/substrate/" env_logger = "0.7.0" futures = "0.1.29" hyper = "0.12.35" -jsonrpc-core-client = { version = "14.0.3", features = ["http", "ws"] } +jsonrpc-core-client = { version = "14.0.5", default-features = false, features = ["http"] } log = "0.4.8" node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } sc-rpc = { version = "2.0.0-alpha.5", path = "../../../client/rpc" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index ada97e566750c5fdc713b2268b69127cab165031..f1d230af90dc5e9e65da0ba4284355fa5292435a 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -25,3 +25,6 @@ sc-keystore = { version = "2.0.0-alpha.5", path = "../../../client/keystore" } sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../../client/consensus/epochs" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 6346f3d8481c0cdd138f7984ac32558887a0fd21..b8e5f706291459f72e3587179054ec8a9e635c72 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } integer-sqrt = { version = "0.1.2" } serde = { version = "1.0.102", optional = true } @@ -55,14 +55,16 @@ pallet-indices = { version = "2.0.0-alpha.5", default-features = false, path = " pallet-identity = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/identity" } pallet-membership = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/membership" } pallet-offences = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/offences" } +pallet-offences-benchmarking = { version = "2.0.0-alpha.5", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } pallet-randomness-collective-flip = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/randomness-collective-flip" } pallet-recovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/recovery" } pallet-session = { version = "2.0.0-alpha.5", features = ["historical"], path = "../../../frame/session", default-features = false } pallet-session-benchmarking = { version = "2.0.0-alpha.5", path = "../../../frame/session/benchmarking", default-features = false, optional = true } pallet-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/staking" } pallet-staking-reward-curve = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/staking/reward-curve" } -pallet-sudo = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/sudo" } +pallet-scheduler = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/scheduler" } pallet-society = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/society" } +pallet-sudo = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/sudo" } pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/timestamp" } pallet-treasury = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/treasury" } pallet-utility = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/utility" } @@ -134,15 +136,24 @@ std = [ runtime-benchmarks = [ "frame-benchmarking", "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", "pallet-balances/runtime-benchmarks", + "pallet-collective/runtime-benchmarks", + "pallet-democracy/runtime-benchmarks", "pallet-elections-phragmen/runtime-benchmarks", "pallet-identity/runtime-benchmarks", + "pallet-im-online/runtime-benchmarks", + "pallet-society/runtime-benchmarks", + "pallet-staking/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", - "pallet-session-benchmarking", - "pallet-staking/runtime-benchmarks", + "pallet-utility/runtime-benchmarks", "pallet-vesting/runtime-benchmarks", + "pallet-collective/runtime-benchmarks", + "pallet-offences-benchmarking", "pallet-session-benchmarking", - "pallet-staking/runtime-benchmarks", - "pallet-im-online/runtime-benchmarks", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 7347db2124afbfc36d3e025f44c44292001adc86..3752ca4d8e534f57a9689db0de29278aeab9d6ed 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -35,7 +35,7 @@ use sp_runtime::{ impl_opaque_keys, generic, create_runtime_str, }; use sp_runtime::curve::PiecewiseLinear; -use sp_runtime::transaction_validity::TransactionValidity; +use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource, TransactionPriority}; use sp_runtime::traits::{ self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys, @@ -59,7 +59,7 @@ pub use pallet_timestamp::Call as TimestampCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_contracts::Gas; pub use frame_support::StorageValue; -pub use pallet_staking::StakerStatus; +pub use pallet_staking::{StakerStatus, LockStakingStatus}; /// Implementations of some helper traits passed into runtime modules as associated types. pub mod impls; @@ -73,6 +73,52 @@ use constants::{time::*, currency::*}; #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); +/// A transaction submitter with the given key type. +pub type TransactionSubmitterOf = TransactionSubmitter; + +/// Submits transaction with the node's public and signature type. Adheres to the signed extension +/// format of the chain. +impl frame_system::offchain::CreateTransaction for Runtime { + type Public = ::Signer; + type Signature = Signature; + + fn create_transaction>( + call: Call, + public: Self::Public, + account: AccountId, + index: Index, + ) -> Option<(Call, ::SignaturePayload)> { + // take the biggest period possible. + let period = BlockHashCount::get() + .checked_next_power_of_two() + .map(|c| c / 2) + .unwrap_or(2) as u64; + let current_block = System::block_number() + .saturated_into::() + // The `System::block_number` is initialized with `n+1`, + // so the actual block number is `n`. + .saturating_sub(1); + let tip = 0; + let extra: SignedExtra = ( + frame_system::CheckVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckEra::::from(generic::Era::mortal(period, current_block)), + frame_system::CheckNonce::::from(index), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from(tip), + Default::default(), + Default::default(), + ); + let raw_payload = SignedPayload::new(call, extra).map_err(|e| { + debug::warn!("Unable to create signed payload: {:?}", e); + }).ok()?; + let signature = TSigner::sign(public, &raw_payload)?; + let address = Indices::unlookup(account); + let (call, extra, _) = raw_payload.deconstruct(); + Some((call, (address, signature, extra))) + } +} + /// Runtime version. pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("node"), @@ -161,6 +207,17 @@ impl pallet_utility::Trait for Runtime { type MaxSignatories = MaxSignatories; } +parameter_types! { + pub const MaximumWeight: Weight = 2_000_000; +} + +impl pallet_scheduler::Trait for Runtime { + type Event = Event; + type Origin = Origin; + type Call = Call; + type MaximumWeight = MaximumWeight; +} + parameter_types! { pub const EpochDuration: u64 = EPOCH_DURATION_IN_SLOTS; pub const ExpectedBlockTime: Moment = MILLISECS_PER_BLOCK; @@ -216,6 +273,7 @@ impl pallet_transaction_payment::Trait for Runtime { parameter_types! { pub const MinimumPeriod: Moment = SLOT_DURATION / 2; } + impl pallet_timestamp::Trait for Runtime { type Moment = Moment; type OnTimestampSet = Babe; @@ -255,6 +313,7 @@ impl pallet_session::Trait for Runtime { type SessionHandler = ::KeyTypeIdProviders; type Keys = SessionKeys; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = Babe; } impl pallet_session::historical::Trait for Runtime { @@ -278,12 +337,13 @@ parameter_types! { pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; + pub const ElectionLookahead: BlockNumber = 25; // 10 minutes per session => 100 block. pub const MaxNominatorRewardedPerValidator: u32 = 64; } impl pallet_staking::Trait for Runtime { type Currency = Balances; - type Time = Timestamp; + type UnixTime = Timestamp; type CurrencyToVote = CurrencyToVoteHandler; type RewardRemainder = Treasury; type Event = Event; @@ -296,7 +356,12 @@ impl pallet_staking::Trait for Runtime { type SlashCancelOrigin = pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>; type SessionInterface = Self; type RewardCurve = RewardCurve; + type NextNewSession = Session; + type ElectionLookahead = ElectionLookahead; + type Call = Call; + type SubmitTransaction = TransactionSubmitterOf<()>; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = StakingUnsignedPriority; } parameter_types! { @@ -340,6 +405,7 @@ impl pallet_democracy::Trait for Runtime { type CooloffPeriod = CooloffPeriod; type PreimageByteDeposit = PreimageByteDeposit; type Slash = Treasury; + type Scheduler = Scheduler; } parameter_types! { @@ -366,6 +432,9 @@ impl pallet_elections_phragmen::Trait for Runtime { type Event = Event; type Currency = Balances; type ChangeMembers = Council; + // NOTE: this implies that council's genesis members cannot be set directly and must come from + // this module. + type InitializeMembers = Council; type CurrencyToVote = CurrencyToVoteHandler; type CandidacyBond = CandidacyBond; type VotingBond = VotingBond; @@ -470,20 +539,21 @@ impl pallet_sudo::Trait for Runtime { type Call = Call; } -/// A runtime transaction submitter. -pub type SubmitTransaction = TransactionSubmitter; - parameter_types! { pub const SessionDuration: BlockNumber = EPOCH_DURATION_IN_SLOTS as _; + pub const ImOnlineUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); + /// We prioritize im-online heartbeats over phragmen solution submission. + pub const StakingUnsignedPriority: TransactionPriority = TransactionPriority::max_value() / 2; } impl pallet_im_online::Trait for Runtime { type AuthorityId = ImOnlineId; type Event = Event; type Call = Call; - type SubmitTransaction = SubmitTransaction; + type SubmitTransaction = TransactionSubmitterOf; type SessionDuration = SessionDuration; type ReportUnresponsiveness = Offences; + type UnsignedPriority = ImOnlineUnsignedPriority; } impl pallet_offences::Trait for Runtime { @@ -530,46 +600,6 @@ impl pallet_identity::Trait for Runtime { type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; } -impl frame_system::offchain::CreateTransaction for Runtime { - type Public = ::Signer; - type Signature = Signature; - - fn create_transaction>( - call: Call, - public: Self::Public, - account: AccountId, - index: Index, - ) -> Option<(Call, ::SignaturePayload)> { - // take the biggest period possible. - let period = BlockHashCount::get() - .checked_next_power_of_two() - .map(|c| c / 2) - .unwrap_or(2) as u64; - let current_block = System::block_number() - .saturated_into::() - // The `System::block_number` is initialized with `n+1`, - // so the actual block number is `n`. - .saturating_sub(1); - let tip = 0; - let extra: SignedExtra = ( - frame_system::CheckVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckEra::::from(generic::Era::mortal(period, current_block)), - frame_system::CheckNonce::::from(index), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - Default::default(), - ); - let raw_payload = SignedPayload::new(call, extra).map_err(|e| { - debug::warn!("Unable to create signed payload: {:?}", e); - }).ok()?; - let signature = TSigner::sign(public, &raw_payload)?; - let address = Indices::unlookup(account); - let (call, extra, _) = raw_payload.deconstruct(); - Some((call, (address, signature, extra))) - } -} - parameter_types! { pub const ConfigDepositBase: Balance = 5 * DOLLARS; pub const FriendDepositFactor: Balance = 50 * CENTS; @@ -638,12 +668,12 @@ construct_runtime!( Indices: pallet_indices::{Module, Call, Storage, Config, Event}, Balances: pallet_balances::{Module, Call, Storage, Config, Event}, TransactionPayment: pallet_transaction_payment::{Module, Storage}, - Staking: pallet_staking::{Module, Call, Config, Storage, Event}, + Staking: pallet_staking::{Module, Call, Config, Storage, Event, ValidateUnsigned}, Session: pallet_session::{Module, Call, Storage, Event, Config}, Democracy: pallet_democracy::{Module, Call, Storage, Config, Event}, Council: pallet_collective::::{Module, Call, Storage, Origin, Event, Config}, TechnicalCommittee: pallet_collective::::{Module, Call, Storage, Origin, Event, Config}, - Elections: pallet_elections_phragmen::{Module, Call, Storage, Event}, + Elections: pallet_elections_phragmen::{Module, Call, Storage, Event, Config}, TechnicalMembership: pallet_membership::::{Module, Call, Storage, Event, Config}, FinalityTracker: pallet_finality_tracker::{Module, Call, Inherent}, Grandpa: pallet_grandpa::{Module, Call, Storage, Config, Event}, @@ -658,6 +688,7 @@ construct_runtime!( Society: pallet_society::{Module, Call, Storage, Event, Config}, Recovery: pallet_recovery::{Module, Call, Storage, Event}, Vesting: pallet_vesting::{Module, Call, Storage, Event, Config}, + Scheduler: pallet_scheduler::{Module, Call, Storage, Event}, } ); @@ -680,6 +711,7 @@ pub type SignedExtra = ( frame_system::CheckWeight, pallet_transaction_payment::ChargeTransactionPayment, pallet_contracts::CheckBlockGasLimit, + pallet_staking::LockStakingStatus, ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; @@ -734,8 +766,11 @@ impl_runtime_apis! { } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction(tx: ::Extrinsic) -> TransactionValidity { - Executive::validate_transaction(tx) + fn validate_transaction( + source: TransactionSource, + tx: ::Extrinsic, + ) -> TransactionValidity { + Executive::validate_transaction(source, tx) } } @@ -845,81 +880,41 @@ impl_runtime_apis! { #[cfg(feature = "runtime-benchmarks")] impl frame_benchmarking::Benchmark for Runtime { fn dispatch_benchmark( - module: Vec, - extrinsic: Vec, + pallet: Vec, + benchmark: Vec, lowest_range_values: Vec, highest_range_values: Vec, steps: Vec, repeat: u32, - ) -> Result, sp_runtime::RuntimeString> { - use frame_benchmarking::Benchmarking; + ) -> Result, sp_runtime::RuntimeString> { + use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark}; // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues. // To get around that, we separated the Session benchmarks into its own crate, which is why // we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; - impl pallet_session_benchmarking::Trait for Runtime {} + use pallet_offences_benchmarking::Module as OffencesBench; - let result = match module.as_slice() { - b"pallet-balances" | b"balances" => Balances::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-identity" | b"identity" => Identity::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-session" | b"session" => SessionBench::::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-staking" | b"staking" => Staking::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-treasury" | b"treasury" => Treasury::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-vesting" | b"vesting" => Vesting::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - _ => Err("Benchmark not found for this pallet."), - }; - - result.map_err(|e| e.into()) + impl pallet_session_benchmarking::Trait for Runtime {} + impl pallet_offences_benchmarking::Trait for Runtime {} + + let mut batches = Vec::::new(); + let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); + + add_benchmark!(params, batches, b"balances", Balances); + add_benchmark!(params, batches, b"collective", Council); + add_benchmark!(params, batches, b"democracy", Democracy); + add_benchmark!(params, batches, b"identity", Identity); + add_benchmark!(params, batches, b"im-online", ImOnline); + add_benchmark!(params, batches, b"session", SessionBench::); + add_benchmark!(params, batches, b"staking", Staking); + add_benchmark!(params, batches, b"timestamp", Timestamp); + add_benchmark!(params, batches, b"treasury", Treasury); + add_benchmark!(params, batches, b"utility", Utility); + add_benchmark!(params, batches, b"vesting", Vesting); + add_benchmark!(params, batches, b"offences", OffencesBench::); + + if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } + Ok(batches) } } } @@ -948,39 +943,7 @@ mod tests { >, {} - is_submit_signed_transaction::(); - is_sign_and_submit_transaction::(); - } - - #[test] - fn block_hooks_weight_should_not_exceed_limits() { - use frame_support::weights::WeighBlock; - let check_for_block = |b| { - let block_hooks_weight = - >::on_initialize(b) + - >::on_finalize(b); - - assert_eq!( - block_hooks_weight, - 0, - "This test might fail simply because the value being compared to has increased to a \ - module declaring a new weight for a hook or call. In this case update the test and \ - happily move on.", - ); - - // Invariant. Always must be like this to have a sane chain. - assert!(block_hooks_weight < MaximumBlockWeight::get()); - - // Warning. - if block_hooks_weight > MaximumBlockWeight::get() / 2 { - println!( - "block hooks weight is consuming more than a block's capacity. You probably want \ - to re-think this. This test will fail now." - ); - assert!(false); - } - }; - - let _ = (0..100_000).for_each(check_for_block); + is_submit_signed_transaction::>(); + is_sign_and_submit_transaction::>(); } } diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index d852aaac260d0761b9bd80883c87d3b07cba023c..df73e20070d2cedca064206d6b6dfc9b796272f2 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -14,7 +14,7 @@ pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } sc-client-db = { version = "0.8.0-alpha.5", path = "../../../client/db/", features = ["kvdb-rocksdb"] } sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } pallet-contracts = { version = "2.0.0-alpha.5", path = "../../../frame/contracts" } pallet-grandpa = { version = "2.0.0-alpha.5", path = "../../../frame/grandpa" } pallet-indices = { version = "2.0.0-alpha.5", path = "../../../frame/indices" } @@ -53,6 +53,5 @@ criterion = "0.3.0" sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service", features = ["rocksdb"] } -[[bench]] -name = "import" -harness = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/testing/benches/import.rs b/bin/node/testing/benches/import.rs deleted file mode 100644 index b36d2e1181b0a2483ee35fc50e43c1d2ebf27522..0000000000000000000000000000000000000000 --- a/bin/node/testing/benches/import.rs +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Block import benchmark. -//! -//! This benchmark is expected to measure block import operation of -//! some more or less full block. -//! -//! As we also want to protect against cold-cache attacks, this -//! benchmark should not rely on any caching (except those that -//! DO NOT depend on user input). Thus block generation should be -//! based on randomized operation. -//! -//! This is supposed to be very simple benchmark and is not subject -//! to much configuring - just block full of randomized transactions. -//! It is not supposed to measure runtime modules weight correctness - -use std::fmt; -use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes}; -use node_primitives::Block; -use sp_runtime::generic::BlockId; -use criterion::{Criterion, criterion_group, criterion_main}; -use sc_client_api::backend::Backend; - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20)); - targets = bench_block_import, bench_account_reaping, bench_account_ed25519 -); -criterion_group!( - name = wasm_size; - config = Criterion::default().sample_size(10); - targets = bench_wasm_size_import -); -criterion_group!( - name = profile; - config = Criterion::default().sample_size(10); - targets = profile_block_import -); -criterion_main!(benches, profile); - -fn bench_block_import(c: &mut Criterion) { - sc_cli::init_logger(""); - // for future uses, uncomment if something wrong. - // sc_cli::init_logger("sc_client=debug"); - - let mut bench_db = BenchDb::new(100); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - log::trace!( - target: "bench-logistics", - "Seed database directory: {}", - bench_db.path().display(), - ); - - c.bench_function_over_inputs("import-block-B-0001", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - - // mostly to just launch compiler before benching! - let version = context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version") - .spec_version; - - log::trace!( - target: "bench-logistics", - "Next iteration database directory: {}, runtime version: {}", - context.path().display(), version, - ); - - context - }, - |mut context| { - let start = std::time::Instant::now(); - context.import_block(block.clone()); - let elapsed = start.elapsed(); - - log::info!( - target: "bench-logistics", - "imported block with {} tx, took: {:#?}", - block.extrinsics.len(), - elapsed, - ); - - log::info!( - target: "bench-logistics", - "usage info: {}", - context.backend.usage_info() - .expect("RocksDB backend always provides usage info!"), - ); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -fn bench_account_reaping(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::new(100); - let block = bench_db.generate_block(BlockType::RandomTransfersReaping(100)); - - c.bench_function_over_inputs("import-block-reaping-B-0002", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - - // mostly to just launch compiler before benching! - context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version"); - - context - }, - |mut context| { - context.import_block(block.clone()); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -fn bench_account_ed25519(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::with_key_types(100, KeyTypes::Ed25519); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - c.bench_function_over_inputs("import-block-ed25519-B-0003", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version"); - - context - }, - |mut context| { - context.import_block(block.clone()); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -// This is not an actual benchmark, so don't use it to measure anything. -// It just produces special pattern of cpu load that allows easy picking -// the part of block import for the profiling in the tool of choice. -fn profile_block_import(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::new(128); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - c.bench_function("profile block", - move |bencher| { - bencher.iter_batched( - || { - bench_db.create_context(Profile::Native) - }, - |mut context| { - // until better osx signpost/callgrind signal is possible to use - // in rust, we just pause everything completely to help choosing - // actual profiling interval - std::thread::park_timeout(std::time::Duration::from_secs(2)); - context.import_block(block.clone()); - // and here as well - std::thread::park_timeout(std::time::Duration::from_secs(2)); - log::info!( - target: "bench-logistics", - "imported block, usage info: {}", - context.backend.usage_info() - .expect("RocksDB backend always provides usage info!"), - ) - }, - criterion::BatchSize::PerIteration, - ); - }, - ); -} - -struct Setup { - db: BenchDb, - block: Block, -} - -struct SetupIterator { - current: usize, - finish: usize, - multiplier: usize, -} - -impl SetupIterator { - fn new(current: usize, finish: usize, multiplier: usize) -> Self { - SetupIterator { current, finish, multiplier } - } -} - -impl Iterator for SetupIterator { - type Item = Setup; - - fn next(&mut self) -> Option { - if self.current >= self.finish { return None } - - self.current += 1; - - let size = self.current * self.multiplier; - let mut db = BenchDb::new(size); - let block = db.generate_block(BlockType::RandomTransfers(size)); - Some(Setup { db, block }) - } -} - -impl fmt::Debug for Setup { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Setup: {} tx/block", self.block.extrinsics.len()) - } -} - -fn bench_wasm_size_import(c: &mut Criterion) { - sc_cli::init_logger(""); - - c.bench_function_over_inputs("wasm_size_import", - move |bencher, setup| { - bencher.iter_batched( - || { - setup.db.create_context(Profile::Wasm) - }, - |mut context| { - context.import_block(setup.block.clone()); - }, - criterion::BatchSize::PerIteration, - ); - }, - SetupIterator::new(5, 15, 50), - ); -} diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 3bc8d483ea2afe4c7cadc0ad9a74e746b14121ea..5dd0d012fbf46b24af4381ee4699c997f4e89cb1 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -186,7 +186,7 @@ impl BenchDb { pruning: PruningMode::ArchiveAll, source: sc_client_db::DatabaseSettingsSrc::Path { path: dir.into(), - cache_size: None, + cache_size: 128, }, }; diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index e35059e0c6ffd300cb2941e4322e22c8bbf832ee..8a57010770f3dedf9795e0da462455943fb6f74d 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -109,6 +109,7 @@ pub fn config_endowed( pallet_collective_Instance1: Some(Default::default()), pallet_collective_Instance2: Some(Default::default()), pallet_membership_Instance1: Some(Default::default()), + pallet_elections_phragmen: Some(Default::default()), pallet_sudo: Some(Default::default()), pallet_treasury: Some(Default::default()), pallet_society: Some(SocietyConfig { diff --git a/bin/node/testing/src/keyring.rs b/bin/node/testing/src/keyring.rs index 6b0d06875d6924a0715f74334390193736009bd0..5fa1e48b0321813d3c9bec039ec3212c5be4216a 100644 --- a/bin/node/testing/src/keyring.rs +++ b/bin/node/testing/src/keyring.rs @@ -75,6 +75,7 @@ pub fn signed_extra(nonce: Index, extra_fee: Balance) -> SignedExtra { frame_system::CheckWeight::new(), pallet_transaction_payment::ChargeTransactionPayment::from(extra_fee), Default::default(), + Default::default(), ) } diff --git a/bin/node/transaction-factory/Cargo.toml b/bin/node/transaction-factory/Cargo.toml index b95b55a53620d66ecabd901bcb62045b15fd0824..33ebeb767ae37e75c04309c958e3de79096bee67 100644 --- a/bin/node/transaction-factory/Cargo.toml +++ b/bin/node/transaction-factory/Cargo.toml @@ -13,7 +13,7 @@ sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } sc-client = { version = "0.8.0-alpha.5", path = "../../../client" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } log = "0.4.8" sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } @@ -21,3 +21,6 @@ sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/service" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 0cadbe76144f59ea1805463373a86f584bd8ad0f..9b03dd17b76de499094287fc126a0607c866a3a3 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -15,3 +15,6 @@ node-cli = { version = "2.0.0-alpha.5", path = "../../node/cli" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } rand = "0.7.2" structopt = "0.3.8" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index 3673909706cdb765508e4e72230d66c0ffc46635..5aa7e7fff71797cdd6aecff053cae4cf223e9c9f 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -87,7 +87,7 @@ fn genesis_constructor( let authorities = authority_seeds .iter() .map(AsRef::as_ref) - .map(chain_spec::get_authority_keys_from_seed) + .map(chain_spec::authority_keys_from_seed) .collect::>(); let enable_println = true; @@ -142,7 +142,7 @@ fn generate_authority_keys_and_store( ).map_err(|err| err.to_string())?; let (_, _, grandpa, babe, im_online, authority_discovery) = - chain_spec::get_authority_keys_from_seed(seed); + chain_spec::authority_keys_from_seed(seed); let insert_key = |key_type, public| { keystore.write().insert_unknown( diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index d8d7532a4c16c95b0aa89cdb353cdbf2690f0674..672f25275c74fbfbab2c1f0908cfbefef8afed42 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -19,7 +19,7 @@ tiny-bip39 = "0.7" substrate-bip39 = "0.4.1" hex = "0.4.0" hex-literal = "0.2.1" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } frame-system = { version = "2.0.0-alpha.5", path = "../../../frame/system" } pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" } pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment" } @@ -29,8 +29,11 @@ derive_more = { version = "0.99.2" } sc-rpc = { version = "2.0.0-alpha.5", path = "../../../client/rpc" } jsonrpc-core-client = { version = "14.0.3", features = ["http"] } hyper = "0.12.35" -libp2p = "0.16.2" +libp2p = "0.17.0" serde_json = "1.0" [features] bench = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index bc7d7a602abca61edb9a8b96800e4f1c1f1ac04c..237cc68df2f4657808152d493d2e26ac9866231f 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -79,6 +79,7 @@ trait Crypto: Sized { ) where ::Public: PublicT, { + let v = network_override.unwrap_or_default(); if let Ok((pair, seed)) = Self::Pair::from_phrase(uri, password) { let public_key = Self::public_from_pair(&pair); @@ -86,6 +87,7 @@ trait Crypto: Sized { OutputType::Json => { let json = json!({ "secretPhrase": uri, + "networkId": String::from(v), "secretSeed": format_seed::(seed), "publicKey": format_public_key::(public_key.clone()), "accountId": format_account_id::(public_key), @@ -95,11 +97,13 @@ trait Crypto: Sized { }, OutputType::Text => { println!("Secret phrase `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", + Network ID/version: {}\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", uri, + String::from(v), format_seed::(seed), format_public_key::(public_key.clone()), format_account_id::(public_key), @@ -114,6 +118,7 @@ trait Crypto: Sized { OutputType::Json => { let json = json!({ "secretKeyUri": uri, + "networkId": String::from(v), "secretSeed": if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, "publicKey": format_public_key::(public_key.clone()), "accountId": format_account_id::(public_key), @@ -123,11 +128,13 @@ trait Crypto: Sized { }, OutputType::Text => { println!("Secret Key URI `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", + Network ID/version: {}\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", uri, + String::from(v), if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, format_public_key::(public_key.clone()), format_account_id::(public_key), @@ -683,6 +690,7 @@ fn create_extrinsic( frame_system::CheckWeight::::new(), pallet_transaction_payment::ChargeTransactionPayment::::from(f), Default::default(), + Default::default(), ) }; let raw_payload = SignedPayload::from_raw( @@ -696,6 +704,7 @@ fn create_extrinsic( (), (), (), + (), ), ); let signature = raw_payload.using_encoded(|payload| signer.sign(payload)).into_runtime(); diff --git a/client/Cargo.toml b/client/Cargo.toml index 7a38faad04e0fc3ed3b08173c2b39daed9a515d9..5d4b5927111ea220c33256c50c8250fe198ec840 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -11,7 +11,7 @@ description = "Substrate Client and associated logic." [dependencies] sc-block-builder = { version = "0.8.0-alpha.5", path = "block-builder" } sc-client-api = { version = "2.0.0-alpha.5", path = "api" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } sp-consensus = { version = "0.8.0-alpha.5", path = "../primitives/consensus/common" } derive_more = { version = "0.99.2" } sc-executor = { version = "0.8.0-alpha.5", path = "executor" } @@ -30,6 +30,7 @@ sp-std = { version = "2.0.0-alpha.5", path = "../primitives/std" } sp-version = { version = "2.0.0-alpha.5", path = "../primitives/version" } sp-api = { version = "2.0.0-alpha.5", path = "../primitives/api" } sp-runtime = { version = "2.0.0-alpha.5", path = "../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../primitives/utils" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../primitives/blockchain" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../primitives/state-machine" } sc-telemetry = { version = "2.0.0-alpha.5", path = "telemetry" } @@ -43,3 +44,6 @@ tempfile = "3.1.0" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../test-utils/runtime/client" } kvdb-memorydb = "0.5.0" sp-panic-handler = { version = "2.0.0-alpha.5", path = "../primitives/panic-handler" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index c3afdee41394e1cf8833e80acc8b6c80fd5fe756..dbe6afe9c7cae00c99abca8464dd4acd4c4f8a3c 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/sc-client-api" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } derive_more = { version = "0.99.2" } sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } @@ -26,10 +26,12 @@ sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } kvdb = "0.5.0" log = { version = "0.4.8" } parking_lot = "0.10.0" +lazy_static = "1.4.0" sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } @@ -39,3 +41,6 @@ sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/tran [dev-dependencies] sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 06d49da640da6158dd373dd4ab3072c8639e9534..c855cd3a0832883c3a66b5c986871909aece048a 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -17,7 +17,6 @@ //! A set of APIs supported by the client along with their primitives. use std::{fmt, collections::HashSet}; -use futures::channel::mpsc; use sp_core::storage::StorageKey; use sp_runtime::{ traits::{Block as BlockT, NumberFor}, @@ -28,13 +27,14 @@ use sp_consensus::BlockOrigin; use crate::blockchain::Info; use crate::notifications::StorageEventStream; +use sp_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain; /// Type that implements `futures::Stream` of block import events. -pub type ImportNotifications = mpsc::UnboundedReceiver>; +pub type ImportNotifications = TracingUnboundedReceiver>; /// A stream of block finality notifications. -pub type FinalityNotifications = mpsc::UnboundedReceiver>; +pub type FinalityNotifications = TracingUnboundedReceiver>; /// Expected hashes of blocks at given heights. /// @@ -179,8 +179,12 @@ pub struct IoInfo { pub state_reads: u64, /// State reads (keys) from cache. pub state_reads_cache: u64, - /// State reads (keys) from cache. + /// State reads (keys) pub state_writes: u64, + /// State write (keys) already cached. + pub state_writes_cache: u64, + /// State write (trie nodes) to backend db. + pub state_writes_nodes: u64, } /// Usage statistics for running client instance. @@ -202,7 +206,7 @@ impl fmt::Display for UsageInfo { f, "caches: ({} state, {} db overlay), \ state db: ({} non-canonical, {} pruning, {} pinned), \ - i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)", + i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} trie nodes writes)", self.memory.state_cache, self.memory.database_cache, self.memory.state_db.non_canonical, @@ -214,7 +218,7 @@ impl fmt::Display for UsageInfo { self.io.average_transaction_size, self.io.state_reads_cache, self.io.state_reads, - self.io.state_writes, + self.io.state_writes_nodes, ) } } diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index 72a9f357fce337c7dec32758ff5aa2d0a8b6bd6a..f154eade44d5e37cd76a3bad97e21d46e3ed779d 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -22,9 +22,9 @@ use std::{ }; use fnv::{FnvHashSet, FnvHashMap}; -use futures::channel::mpsc; use sp_core::storage::{StorageKey, StorageData}; use sp_runtime::traits::Block as BlockT; +use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded}; /// Storage change set #[derive(Debug)] @@ -67,7 +67,7 @@ impl StorageChangeSet { } /// Type that implements `futures::Stream` of storage change events. -pub type StorageEventStream = mpsc::UnboundedReceiver<(H, StorageChangeSet)>; +pub type StorageEventStream = TracingUnboundedReceiver<(H, StorageChangeSet)>; type SubscriberId = u64; @@ -82,7 +82,7 @@ pub struct StorageNotifications { FnvHashSet )>, sinks: FnvHashMap, + TracingUnboundedSender<(Block::Hash, StorageChangeSet)>, Option>, Option>>>, )>, @@ -299,7 +299,7 @@ impl StorageNotifications { // insert sink - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_storage_notification_items"); self.sinks.insert(current_id, (tx, keys, child_keys)); rx } diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 5a7f78b95d34c036a1b624eed48136ecd1cc26aa..7521101ae6f2a2f0e1df77bb25df381c45e3bf66 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -14,11 +14,11 @@ prost-build = "0.6.1" [dependencies] bytes = "0.5.0" -codec = { package = "parity-scale-codec", default-features = false, version = "1.2.0" } +codec = { package = "parity-scale-codec", default-features = false, version = "1.3.0" } derive_more = "0.99.2" -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.16.2", default-features = false, features = ["secp256k1", "libp2p-websocket"] } +libp2p = { version = "0.17.0", default-features = false, features = ["secp256k1", "libp2p-websocket"] } log = "0.4.8" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.5"} prost = "0.6.1" @@ -38,3 +38,6 @@ env_logger = "0.7.0" quickcheck = "0.9.0" sc-peerset = { version = "2.0.0-alpha.5", path = "../peerset" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client"} + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/authority-discovery/src/error.rs b/client/authority-discovery/src/error.rs index 74b7043c29ba0c3fb8439c6dde47c619183831ed..216f8c26bf07e397a80612c17ca8ea82cd2275da 100644 --- a/client/authority-discovery/src/error.rs +++ b/client/authority-discovery/src/error.rs @@ -16,6 +16,8 @@ //! Authority discovery errors. +use sp_core::crypto::CryptoTypePublicPair; + /// AuthorityDiscovery Result. pub type Result = std::result::Result; @@ -46,6 +48,10 @@ pub enum Error { EncodingDecodingScale(codec::Error), /// Failed to parse a libp2p multi address. ParsingMultiaddress(libp2p::core::multiaddr::Error), + /// Failed to sign using a specific public key + MissingSignature(CryptoTypePublicPair), + /// Failed to sign using all public keys + Signing, /// Failed to register Prometheus metric. Prometheus(prometheus_endpoint::PrometheusError), } diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index 171a401bbbf8b5d00bb80d06ee0dbe2ad7d482ce..45f05e1039a5bdc1c5e8eb5355117ff5a21719d4 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -56,14 +56,13 @@ use futures_timer::Delay; use codec::{Decode, Encode}; use error::{Error, Result}; -use libp2p::Multiaddr; use log::{debug, error, log_enabled, warn}; use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register}; use prost::Message; use sc_client_api::blockchain::HeaderBackend; -use sc_network::{DhtEvent, ExHashT, NetworkStateInfo}; +use sc_network::{Multiaddr, config::MultiaddrWithPeerId, DhtEvent, ExHashT, NetworkStateInfo}; use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; -use sp_core::crypto::{key_types, Pair}; +use sp_core::crypto::{key_types, CryptoTypePublicPair, Pair}; use sp_core::traits::BareCryptoStorePtr; use sp_runtime::{traits::Block as BlockT, generic::BlockId}; use sp_api::ProvideRuntimeApi; @@ -187,7 +186,7 @@ where pub fn new( client: Arc, network: Arc, - sentry_nodes: Vec, + sentry_nodes: Vec, key_store: BareCryptoStorePtr, dht_event_rx: Pin + Send>>, prometheus_registry: Option, @@ -210,18 +209,7 @@ where ); let sentry_nodes = if !sentry_nodes.is_empty() { - let addrs = sentry_nodes.into_iter().filter_map(|a| match a.parse() { - Ok(addr) => Some(addr), - Err(e) => { - error!( - target: "sub-authority-discovery", - "Failed to parse sentry node public address '{:?}', continuing anyways.", e, - ); - None - } - }).collect::>(); - - Some(addrs) + Some(sentry_nodes.into_iter().map(|ma| ma.concat()).collect::>()) } else { None }; @@ -283,19 +271,36 @@ where .encode(&mut serialized_addresses) .map_err(Error::EncodingProto)?; - for key in self.get_priv_keys_within_authority_set()?.into_iter() { - let signature = key.sign(&serialized_addresses); + let keys: Vec = self.get_own_public_keys_within_authority_set()? + .into_iter() + .map(Into::into) + .collect(); + let signatures = self.key_store + .read() + .sign_with_all( + key_types::AUTHORITY_DISCOVERY, + keys.clone(), + serialized_addresses.as_slice(), + ) + .map_err(|_| Error::Signing)?; + + for (sign_result, key) in signatures.iter().zip(keys) { let mut signed_addresses = vec![]; + + // sign_with_all returns Result signature + // is generated for a public key that is supported. + // Verify that all signatures exist for all provided keys. + let signature = sign_result.as_ref().map_err(|_| Error::MissingSignature(key.clone()))?; schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), - signature: signature.encode(), + signature: Encode::encode(&signature), } .encode(&mut signed_addresses) .map_err(Error::EncodingProto)?; self.network.put_value( - hash_authority_id(key.public().as_ref())?, + hash_authority_id(key.1.as_ref()), signed_addresses, ); } @@ -318,7 +323,7 @@ where for authority_id in authorities.iter() { self.network - .get_value(&hash_authority_id(authority_id.as_ref())?); + .get_value(&hash_authority_id(authority_id.as_ref())); } Ok(()) @@ -403,8 +408,8 @@ where self.addr_cache.retain_ids(&authorities); authorities .into_iter() - .map(|id| hash_authority_id(id.as_ref()).map(|h| (h, id))) - .collect::>>()? + .map(|id| (hash_authority_id(id.as_ref()), id)) + .collect::>() }; // Check if the event origins from an authority in the current authority set. @@ -446,21 +451,6 @@ where Ok(()) } - /// Retrieve all local authority discovery private keys that are within the current authority - /// set. - fn get_priv_keys_within_authority_set(&mut self) -> Result> { - let keys = self.get_own_public_keys_within_authority_set()? - .into_iter() - .map(std::convert::Into::into) - .filter_map(|pub_key| { - self.key_store.read().sr25519_key_pair(key_types::AUTHORITY_DISCOVERY, &pub_key) - }) - .map(std::convert::Into::into) - .collect(); - - Ok(keys) - } - /// Retrieve our public keys within the current authority set. // // A node might have multiple authority discovery keys within its keystore, e.g. an old one and @@ -596,10 +586,8 @@ where } } -fn hash_authority_id(id: &[u8]) -> Result { - libp2p::multihash::encode(libp2p::multihash::Hash::SHA2256, id) - .map(|k| libp2p::kad::record::Key::new(&k)) - .map_err(Error::HashingAuthorityId) +fn hash_authority_id(id: &[u8]) -> libp2p::kad::record::Key { + libp2p::kad::record::Key::new(&libp2p::multihash::Sha2_256::digest(id)) } fn interval_at(start: Instant, duration: Duration) -> Interval { diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs index 1296c2b627a1eab7c966744ae79d3bf2557e1456..358376e5db471f4aa218b9aa13ed700b7544ac95 100644 --- a/client/authority-discovery/src/tests.rs +++ b/client/authority-discovery/src/tests.rs @@ -21,8 +21,8 @@ use futures::executor::block_on; use futures::future::poll_fn; use libp2p::{kad, PeerId}; -use sp_api::{ApiExt, ApiErrorExt, Core, RuntimeVersion, StorageProof, ProvideRuntimeApi, ApiRef}; -use sp_core::{testing::KeyStore, ExecutionContext, NativeOrEncoded}; +use sp_api::{ProvideRuntimeApi, ApiRef}; +use sp_core::testing::KeyStore; use sp_runtime::traits::{Zero, Block as BlockT, NumberFor}; use substrate_test_runtime_client::runtime::Block; @@ -99,8 +99,7 @@ impl ProvideRuntimeApi for TestApi { fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> { RuntimeApi { authorities: self.authorities.clone(), - } - .into() + }.into() } } @@ -120,6 +119,7 @@ impl HeaderBackend for TestApi { finalized_hash: Default::default(), finalized_number: Zero::zero(), genesis_hash: Default::default(), + number_leaves: Default::default(), } } @@ -149,90 +149,13 @@ struct RuntimeApi { authorities: Vec, } -impl Core for RuntimeApi { - fn Core_version_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<()>, - _: Vec, - ) -> std::result::Result, sp_blockchain::Error> { - unimplemented!("Not required for testing!") - } - - fn Core_execute_block_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option, - _: Vec, - ) -> std::result::Result, sp_blockchain::Error> { - unimplemented!("Not required for testing!") - } +sp_api::mock_impl_runtime_apis! { + impl AuthorityDiscoveryApi for RuntimeApi { + type Error = sp_blockchain::Error; - fn Core_initialize_block_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<&::Header>, - _: Vec, - ) -> std::result::Result, sp_blockchain::Error> { - unimplemented!("Not required for testing!") - } -} - -impl ApiErrorExt for RuntimeApi { - type Error = sp_blockchain::Error; -} - -impl ApiExt for RuntimeApi { - type StateBackend = < - substrate_test_runtime_client::Backend as sc_client_api::backend::Backend - >::State; - - fn map_api_result std::result::Result, R, E>( - &self, - _: F - ) -> std::result::Result { - unimplemented!("Not required for testing!") - } - - fn runtime_version_at( - &self, - _: &BlockId, - ) -> std::result::Result { - unimplemented!("Not required for testing!") - } - - fn record_proof(&mut self) { - unimplemented!("Not required for testing!") - } - - fn extract_proof(&mut self) -> Option { - unimplemented!("Not required for testing!") - } - - fn into_storage_changes( - &self, - _: &Self::StateBackend, - _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, - _: ::Hash, - ) -> std::result::Result, String> - where Self: Sized - { - unimplemented!("Not required for testing!") - } -} - -impl AuthorityDiscoveryApi for RuntimeApi { - fn AuthorityDiscoveryApi_authorities_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<()>, - _: Vec, - ) -> std::result::Result>, sp_blockchain::Error> { - return Ok(NativeOrEncoded::Native(self.authorities.clone())); + fn authorities(&self) -> Vec { + self.authorities.clone() + } } } @@ -381,7 +304,7 @@ fn handle_dht_events_with_value_found_should_call_set_priority_group() { // Create sample dht event. - let authority_id_1 = hash_authority_id(key_pair.public().as_ref()).unwrap(); + let authority_id_1 = hash_authority_id(key_pair.public().as_ref()); let address_1: Multiaddr = "/ip6/2001:db8::".parse().unwrap(); let mut serialized_addresses = vec![]; diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 34ce9c948c01171f4655b9441e20ac1e841d4897..040370ac491cf09932ef1e7467ebeb96847b5af8 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -10,8 +10,8 @@ description = "Basic implementation of block-authoring logic." [dependencies] log = "0.4.8" -futures = "0.3.1" -codec = { package = "parity-scale-codec", version = "1.2.0" } +futures = "0.3.4" +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } @@ -29,3 +29,6 @@ futures-timer = "3.0.1" sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../client/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } parking_lot = "0.10.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 986d797ca736b8cd5e9c2e2a09efb02924b06603..37bb34a4b67d993fb02bc6b1e5229db0c7fa7f81 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -23,7 +23,7 @@ use sc_client_api::backend; use codec::Decode; use sp_consensus::{evaluation, Proposal, RecordProof}; use sp_inherents::InherentData; -use log::{error, info, debug, trace}; +use log::{error, info, debug, trace, warn}; use sp_core::ExecutionContext; use sp_runtime::{ generic::BlockId, @@ -34,7 +34,7 @@ use sc_telemetry::{telemetry, CONSENSUS_INFO}; use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; use sp_api::{ProvideRuntimeApi, ApiExt}; use futures::{executor, future, future::Either}; -use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed}; +use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed}; use std::marker::PhantomData; /// Proposer factory. @@ -76,7 +76,7 @@ impl ProposerFactory let id = BlockId::hash(parent_hash); - info!("Starting consensus session on top of parent {:?}", parent_hash); + info!("🙌 Starting consensus session on top of parent {:?}", parent_hash); let proposer = Proposer { inner: Arc::new(ProposerInner { @@ -196,14 +196,25 @@ impl ProposerInner // We don't check the API versions any further here since the dispatch compatibility // check should be enough. - for extrinsic in self.client.runtime_api() + for inherent in self.client.runtime_api() .inherent_extrinsics_with_context( &self.parent_id, ExecutionContext::BlockConstruction, inherent_data )? { - block_builder.push(extrinsic)?; + match block_builder.push(inherent) { + Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => + warn!("⚠️ Dropping non-mandatory inherent from overweight block."), + Err(ApplyExtrinsicFailed(Validity(e))) if e.was_mandatory() => { + error!("❌️ Mandatory inherent extrinsic returned error. Block cannot be produced."); + Err(ApplyExtrinsicFailed(Validity(e)))? + } + Err(e) => { + warn!("❗️ Inherent extrinsic returned unexpected error: {}. Dropping.", e); + } + Ok(_) => {} + } } // proceed with transactions @@ -212,7 +223,7 @@ impl ProposerInner let mut unqueue_invalid = Vec::new(); let pending_iterator = match executor::block_on(future::select( self.transaction_pool.ready_at(self.parent_number), - futures_timer::Delay::new((deadline - (self.now)()) / 8), + futures_timer::Delay::new(deadline.saturating_duration_since((self.now)()) / 8), )) { Either::Left((iterator, _)) => iterator, Either::Right(_) => { @@ -241,7 +252,7 @@ impl ProposerInner Ok(()) => { debug!("[{:?}] Pushed to the block.", pending_tx_hash); } - Err(sp_blockchain::Error::ApplyExtrinsicFailed(ApplyExtrinsicFailed::Validity(e))) + Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { if is_first { debug!("[{:?}] Invalid transaction: FullBlock on empty block", pending_tx_hash); @@ -277,7 +288,7 @@ impl ProposerInner let (block, storage_changes, proof) = block_builder.build()?.into_inner(); - info!("Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", + info!("🎁 Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", block.header().number(), ::Hash::from(block.header().hash()), block.header().parent_hash(), @@ -315,13 +326,15 @@ mod tests { prelude::*, runtime::{Extrinsic, Transfer}, }; - use sp_transaction_pool::{ChainEvent, MaintainedTransactionPool}; + use sp_transaction_pool::{ChainEvent, MaintainedTransactionPool, TransactionSource}; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_api::Core; use backend::Backend; use sp_blockchain::HeaderBackend; use sp_runtime::traits::NumberFor; + const SOURCE: TransactionSource = TransactionSource::External; + fn extrinsic(nonce: u64) -> Extrinsic { Transfer { amount: Default::default(), @@ -338,7 +351,7 @@ mod tests { id: BlockId::Number(block_number.into()), retracted: vec![], is_new_best: true, - header: header, + header, } } @@ -351,7 +364,7 @@ mod tests { ); futures::executor::block_on( - txpool.submit_at(&BlockId::number(0), vec![extrinsic(0), extrinsic(1)]) + txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0), extrinsic(1)]) ).unwrap(); futures::executor::block_on( @@ -391,6 +404,36 @@ mod tests { assert_eq!(txpool.ready().count(), 2); } + #[test] + fn should_not_panic_when_deadline_is_reached() { + let client = Arc::new(substrate_test_runtime_client::new()); + let txpool = Arc::new( + BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + ); + + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + + let cell = Mutex::new((false, time::Instant::now())); + let mut proposer = proposer_factory.init_with_now( + &client.header(&BlockId::number(0)).unwrap().unwrap(), + Box::new(move || { + let mut value = cell.lock(); + if !value.0 { + value.0 = true; + return value.1; + } + let new = value.1 + time::Duration::from_secs(160); + *value = (true, new); + new + }) + ); + + let deadline = time::Duration::from_secs(1); + futures::executor::block_on( + proposer.propose(Default::default(), Default::default(), deadline, RecordProof::No) + ).map(|r| r.block).unwrap(); + } + #[test] fn proposed_storage_changes_should_match_execute_block_storage_changes() { let (client, backend) = substrate_test_runtime_client::TestClientBuilder::new() @@ -403,7 +446,7 @@ mod tests { let block_id = BlockId::Hash(genesis_hash); futures::executor::block_on( - txpool.submit_at(&BlockId::number(0), vec![extrinsic(0)]), + txpool.submit_at(&BlockId::number(0), SOURCE, vec![extrinsic(0)]), ).unwrap(); futures::executor::block_on( @@ -454,7 +497,7 @@ mod tests { ); futures::executor::block_on( - txpool.submit_at(&BlockId::number(0), vec![ + txpool.submit_at(&BlockId::number(0), SOURCE, vec![ extrinsic(0), extrinsic(1), Transfer { diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 351c3e0a77943e6849cd253f86e4266a735096db..5f9ef7c75e8591baaada2b2ff1a46fbebc72f231 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -18,8 +18,11 @@ sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-block-builder = { version = "2.0.0-alpha.5", path = "../../primitives/block-builder" } sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } [dev-dependencies] substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 2eb172ffeefe56e6231599d20dacf984f3f35098..5d65cbd842b5430064ae2249fe63f9f0b4cf2b8c 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -17,3 +17,6 @@ serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index f7e57e26f36ac4705e5e792dff3d2b8214c5ecc2..9343c9a6de1c95919abe695b4918bd576117efbf 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -18,3 +18,6 @@ quote = "1.0.3" syn = "1.0.7" [dev-dependencies] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 33f217cb4e7d86b71a600d75a898d5010c564d35..a7cf1d8aac4143ff758ec67b39e32c39eb6cc379 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -27,7 +27,7 @@ use sp_runtime::BuildStorage; use serde_json as json; use crate::RuntimeGenesis; use crate::extension::GetExtension; -use sc_network::Multiaddr; +use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; enum GenesisSource { @@ -137,7 +137,7 @@ enum Genesis { struct ClientSpec { name: String, id: String, - boot_nodes: Vec, + boot_nodes: Vec, telemetry_endpoints: Option, protocol_id: Option, properties: Option, @@ -174,7 +174,7 @@ impl Clone for ChainSpec { impl ChainSpec { /// A list of bootnode addresses. - pub fn boot_nodes(&self) -> &[String] { + pub fn boot_nodes(&self) -> &[MultiaddrWithPeerId] { &self.client_spec.boot_nodes } @@ -206,8 +206,8 @@ impl ChainSpec { } /// Add a bootnode to the list. - pub fn add_boot_node(&mut self, addr: Multiaddr) { - self.client_spec.boot_nodes.push(addr.to_string()) + pub fn add_boot_node(&mut self, addr: MultiaddrWithPeerId) { + self.client_spec.boot_nodes.push(addr) } /// Returns a reference to defined chain spec extensions. @@ -220,7 +220,7 @@ impl ChainSpec { name: &str, id: &str, constructor: F, - boot_nodes: Vec, + boot_nodes: Vec, telemetry_endpoints: Option, protocol_id: Option<&str>, properties: Option, @@ -320,7 +320,7 @@ where G: RuntimeGenesis, E: GetExtension + serde::Serialize + Clone + Send, { - fn boot_nodes(&self) -> &[String] { + fn boot_nodes(&self) -> &[MultiaddrWithPeerId] { ChainSpec::boot_nodes(self) } @@ -344,7 +344,7 @@ where ChainSpec::properties(self) } - fn add_boot_node(&mut self, addr: Multiaddr) { + fn add_boot_node(&mut self, addr: MultiaddrWithPeerId) { ChainSpec::add_boot_node(self, addr) } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 353c386f3679e9e8e711007f249b2c62ba492cfc..b7875fdbb4670ee97fb9ccd14971d3b0e3c3d95a 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -117,7 +117,7 @@ pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; use serde::{Serialize, de::DeserializeOwned}; use sp_runtime::BuildStorage; -use sc_network::Multiaddr; +use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; /// A set of traits for the runtime genesis config. @@ -131,7 +131,7 @@ pub trait ChainSpec: BuildStorage + Send { /// Spec id. fn id(&self) -> &str; /// A list of bootnode addresses. - fn boot_nodes(&self) -> &[String]; + fn boot_nodes(&self) -> &[MultiaddrWithPeerId]; /// Telemetry endpoints (if any) fn telemetry_endpoints(&self) -> &Option; /// Network protocol id. @@ -143,7 +143,7 @@ pub trait ChainSpec: BuildStorage + Send { /// Returns a reference to defined chain spec extensions. fn extensions(&self) -> &dyn GetExtension; /// Add a bootnode to the list. - fn add_boot_node(&mut self, addr: Multiaddr); + fn add_boot_node(&mut self, addr: MultiaddrWithPeerId); /// Return spec as JSON. fn as_json(&self, raw: bool) -> Result; /// Return StorageBuilder for this spec. diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index cc1b90dd2f49819803b334c738c8f2fc730a9a86..89247cf016d100792a6ca49e3882529c165472c4 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -20,7 +20,7 @@ ansi_term = "0.12.1" lazy_static = "1.4.0" app_dirs = "1.2.1" tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } -futures = "0.3.1" +futures = "0.3.4" fdlimit = "0.1.4" serde_json = "1.0.41" sc-informant = { version = "0.8.0-alpha.5", path = "../informant" } @@ -29,6 +29,7 @@ sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } sc-network = { version = "0.8.0-alpha.5", path = "../network" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../service" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } @@ -44,6 +45,9 @@ parity-util-mem = { version = "0.6.0", default-features = false, features = ["pr [target.'cfg(not(target_os = "unknown"))'.dependencies] rpassword = "4.0.1" +[target.'cfg(target_family = "unix")'.dependencies] +nix = "0.17.0" + [dev-dependencies] tempfile = "3.1.0" @@ -51,3 +55,6 @@ tempfile = "3.1.0" wasmtime = [ "sc-service/wasmtime", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 384087bec0dbe4538c7227e36a6dbce7d789586b..f09a8d8d47392b019675235c42f0bab1a8def14e 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -45,7 +45,9 @@ impl WasmExecutionMethod { impl Into for WasmExecutionMethod { fn into(self) -> sc_service::config::WasmExecutionMethod { match self { - WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, + WasmExecutionMethod::Interpreted => { + sc_service::config::WasmExecutionMethod::Interpreted + } #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, #[cfg(not(feature = "wasmtime"))] diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 59f7fbb4e95a5363fd03de5f696241c374ee6737..a01101fa7965563795eb3058bf3be0f752801517 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -14,15 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use log::info; -use sc_network::config::build_multiaddr; -use sc_service::{Configuration, ChainSpec}; - use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; use crate::params::NodeKeyParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::{config::MultiaddrWithPeerId, Configuration}; +use structopt::StructOpt; /// The `build-spec` command used to build a specification. #[derive(Debug, StructOpt, Clone)] @@ -49,22 +48,18 @@ pub struct BuildSpecCmd { impl BuildSpecCmd { /// Run the build-spec command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> { + pub fn run(&self, config: Configuration) -> error::Result<()> { info!("Building chain spec"); - let mut spec = config.chain_spec.expect("`chain_spec` is set to `Some` in `update_config`"); + let mut spec = config.chain_spec; let raw_output = self.raw; if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { let keys = config.network.node_key.into_keypair()?; let peer_id = keys.public().into_peer_id(); - let addr = build_multiaddr![ - Ip4([127, 0, 0, 1]), - Tcp(30333u16), - P2p(peer_id) - ]; + let addr = MultiaddrWithPeerId { + multiaddr: build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(30333u16)], + peer_id, + }; spec.add_boot_node(addr) } @@ -74,25 +69,14 @@ impl BuildSpecCmd { Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - - let net_config_path = config - .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a base_path"); - - self.node_key_params.update_config(&mut config, Some(&net_config_path))?; +impl CliConfiguration for BuildSpecCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn node_key_params(&self) -> Option<&NodeKeyParams> { + Some(&self.node_key_params) } } - diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs index 88248c596925e1e0c43b1d6d3f13b2fc9262ddcd..ac4fe63da950618f25948c9d2aeabe8c96f3e291 100644 --- a/client/cli/src/commands/check_block_cmd.rs +++ b/client/cli/src/commands/check_block_cmd.rs @@ -14,20 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::generic::BlockId; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; use std::str::FromStr; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, Roles, ChainSpec, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; -use sp_runtime::generic::BlockId; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::SharedParams; -use crate::params::ImportParams; /// The `check-block` command used to validate blocks. #[derive(Debug, StructOpt, Clone)] @@ -53,8 +49,8 @@ pub struct CheckBlockCmd { impl CheckBlockCmd { /// Run the check-block command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -65,37 +61,37 @@ impl CheckBlockCmd { <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, ::Hash: std::str::FromStr, { - let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; + let input = if self.input.starts_with("0x") { + &self.input[2..] + } else { + &self.input[..] + }; let block_id = match FromStr::from_str(input) { Ok(hash) => BlockId::hash(hash), Err(_) => match self.input.parse::() { Ok(n) => BlockId::number((n as u32).into()), - Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), - } + Err(_) => { + return Err(error::Error::Input( + "Invalid hash or number specified".into(), + )) + } + }, }; let start = std::time::Instant::now(); - run_until_exit(config, |config| { - Ok(builder(config)?.check_block(block_id)) - })?; + builder(config)?.check_block(block_id).await?; println!("Completed in {} ms.", start.elapsed().as_millis()); Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for CheckBlockCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs index 61a63806d21b83b9c6a9d5e8a17c111913fd4c96..48abd409d6833ee758965cae3ef11ec867266c0c 100644 --- a/client/cli/src/commands/export_blocks_cmd.rs +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -14,22 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::io; -use std::fs; -use std::path::PathBuf; -use std::fmt::Debug; +use crate::error; +use crate::params::{BlockNumber, PruningParams, SharedParams}; +use crate::CliConfiguration; use log::info; -use structopt::StructOpt; use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, - config::DatabaseConfig, Roles, + config::DatabaseConfig, Configuration, ServiceBuilderCommand, }; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::{SharedParams, BlockNumber, PruningParams}; +use std::fmt::Debug; +use std::fs; +use std::io; +use std::path::PathBuf; +use structopt::StructOpt; /// The `export-blocks` command used to export blocks. #[derive(Debug, StructOpt, Clone)] @@ -65,8 +62,8 @@ pub struct ExportBlocksCmd { impl ExportBlocksCmd { /// Run the export-blocks command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -77,9 +74,10 @@ impl ExportBlocksCmd { <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, ::Hash: std::str::FromStr, { - if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { + if let DatabaseConfig::Path { ref path, .. } = &config.database { info!("DB path: {}", path.display()); } + let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); let to = self.to.as_ref().and_then(|t| t.parse().ok()); @@ -90,24 +88,19 @@ impl ExportBlocksCmd { None => Box::new(io::stdout()), }; - run_until_exit(config, |config| { - Ok(builder(config)?.export_blocks(file, from.into(), to, binary)) - }) + builder(config)? + .export_blocks(file, from.into(), to, binary) + .await + .map_err(Into::into) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.pruning_params.update_config(&mut config, Roles::FULL, true)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for ExportBlocksCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) } } diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs index b43407add1c7f3c0b36235539fd4272c36772dfa..ce95640f469ceb3f380f07727f4ed5bc09d431ae 100644 --- a/client/cli/src/commands/import_blocks_cmd.rs +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -14,21 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; -use std::io::{Read, Seek, self}; use std::fs; +use std::io::{self, Read, Seek}; use std::path::PathBuf; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, Roles, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::SharedParams; -use crate::params::ImportParams; /// The `import-blocks` command used to import blocks. #[derive(Debug, StructOpt, Clone)] @@ -59,8 +55,8 @@ impl ReadPlusSeek for T {} impl ImportBlocksCmd { /// Run the import-blocks command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -77,27 +73,22 @@ impl ImportBlocksCmd { let mut buffer = Vec::new(); io::stdin().read_to_end(&mut buffer)?; Box::new(io::Cursor::new(buffer)) - }, + } }; - run_until_exit(config, |config| { - Ok(builder(config)?.import_blocks(file, false)) - }) + builder(config)? + .import_blocks(file, false) + .await + .map_err(Into::into) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for ImportBlocksCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index d87b08f7f49828ef48bb469c016d2ac65e507906..d05a5464b215da887bd53ec229b9379c6c0ceecc 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -14,34 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -mod runcmd; -mod export_blocks_cmd; mod build_spec_cmd; -mod import_blocks_cmd; mod check_block_cmd; -mod revert_cmd; +mod export_blocks_cmd; +mod import_blocks_cmd; mod purge_chain_cmd; +mod revert_cmd; +mod runcmd; -use std::fmt::Debug; -use structopt::StructOpt; - -use sc_service::{ Configuration, ServiceBuilderCommand, ChainSpec }; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; - -pub use crate::commands::runcmd::RunCmd; -pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; pub use crate::commands::build_spec_cmd::BuildSpecCmd; -pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; pub use crate::commands::check_block_cmd::CheckBlockCmd; -pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; +pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; pub use crate::commands::purge_chain_cmd::PurgeChainCmd; - -/// default sub directory to store network config -const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; +pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::runcmd::RunCmd; +use std::fmt::Debug; +use structopt::StructOpt; /// All core commands that are provided by default. /// @@ -51,89 +40,338 @@ const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; #[derive(Debug, Clone, StructOpt)] pub enum Subcommand { /// Build a spec.json file, outputs to stdout. - BuildSpec(build_spec_cmd::BuildSpecCmd), + BuildSpec(BuildSpecCmd), /// Export blocks to a file. - ExportBlocks(export_blocks_cmd::ExportBlocksCmd), + ExportBlocks(ExportBlocksCmd), /// Import blocks from file. - ImportBlocks(import_blocks_cmd::ImportBlocksCmd), + ImportBlocks(ImportBlocksCmd), /// Validate a single block. - CheckBlock(check_block_cmd::CheckBlockCmd), + CheckBlock(CheckBlockCmd), /// Revert chain to the previous state. - Revert(revert_cmd::RevertCmd), + Revert(RevertCmd), /// Remove the whole chain data. - PurgeChain(purge_chain_cmd::PurgeChainCmd), + PurgeChain(PurgeChainCmd), } -impl Subcommand { - /// Get the shared parameters of a `CoreParams` command. - pub fn get_shared_params(&self) -> &SharedParams { - use Subcommand::*; - - match self { - BuildSpec(params) => ¶ms.shared_params, - ExportBlocks(params) => ¶ms.shared_params, - ImportBlocks(params) => ¶ms.shared_params, - CheckBlock(params) => ¶ms.shared_params, - Revert(params) => ¶ms.shared_params, - PurgeChain(params) => ¶ms.shared_params, - } - } +// TODO: move to config.rs? +/// Macro that helps implement CliConfiguration on an enum of subcommand automatically +/// +/// # Example +/// +/// ``` +/// # #[macro_use] extern crate sc_cli; +/// +/// # struct EmptyVariant {} +/// +/// # impl sc_cli::CliConfiguration for EmptyVariant { +/// # fn shared_params(&self) -> &sc_cli::SharedParams { unimplemented!() } +/// # fn chain_id(&self, _: bool) -> sc_cli::Result { Ok("test-chain-id".to_string()) } +/// # } +/// +/// # fn main() { +/// enum Subcommand { +/// Variant1(EmptyVariant), +/// Variant2(EmptyVariant), +/// } +/// +/// substrate_cli_subcommands!( +/// Subcommand => Variant1, Variant2 +/// ); +/// +/// # use sc_cli::CliConfiguration; +/// # assert_eq!(Subcommand::Variant1(EmptyVariant {}).chain_id(false).unwrap(), "test-chain-id"); +/// +/// # } +/// ``` +/// +/// Which will expand to: +/// +/// ```ignore +/// impl CliConfiguration for Subcommand { +/// fn base_path(&self) -> Result> { +/// match self { +/// Subcommand::Variant1(cmd) => cmd.base_path(), +/// Subcommand::Variant2(cmd) => cmd.base_path(), +/// } +/// } +/// +/// fn is_dev(&self) -> Result { +/// match self { +/// Subcommand::Variant1(cmd) => cmd.is_dev(), +/// Subcommand::Variant2(cmd) => cmd.is_dev(), +/// } +/// } +/// +/// // ... +/// } +/// ``` +#[macro_export] +macro_rules! substrate_cli_subcommands { + ($enum:ident => $($variant:ident),*) => { + impl $crate::CliConfiguration for $enum { + fn shared_params(&self) -> &$crate::SharedParams { + match self { + $($enum::$variant(cmd) => cmd.shared_params()),* + } + } - /// Run any `CoreParams` command. - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - match self { - Subcommand::BuildSpec(cmd) => cmd.run(config), - Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), - Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), - Subcommand::CheckBlock(cmd) => cmd.run(config, builder), - Subcommand::PurgeChain(cmd) => cmd.run(config), - Subcommand::Revert(cmd) => cmd.run(config, builder), - } - } + fn import_params(&self) -> Option<&$crate::ImportParams> { + match self { + $($enum::$variant(cmd) => cmd.import_params()),* + } + } - /// Update and prepare a `Configuration` with command line parameters. - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - match self { - Subcommand::BuildSpec(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::ExportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::ImportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::CheckBlock(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::PurgeChain(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::Revert(cmd) => cmd.update_config(&mut config, spec_factory, version), - } - } + fn pruning_params(&self) -> Option<&$crate::PruningParams> { + match self { + $($enum::$variant(cmd) => cmd.pruning_params()),* + } + } + + fn keystore_params(&self) -> Option<&$crate::KeystoreParams> { + match self { + $($enum::$variant(cmd) => cmd.keystore_params()),* + } + } + + fn network_params(&self) -> Option<&$crate::NetworkParams> { + match self { + $($enum::$variant(cmd) => cmd.network_params()),* + } + } + + fn base_path(&self) -> $crate::Result<::std::option::Option<::std::path::PathBuf>> { + match self { + $($enum::$variant(cmd) => cmd.base_path()),* + } + } + + fn is_dev(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.is_dev()),* + } + } + + fn role(&self, is_dev: bool) -> $crate::Result<::sc_service::Role> { + match self { + $($enum::$variant(cmd) => cmd.role(is_dev)),* + } + } + + fn transaction_pool(&self) + -> $crate::Result<::sc_service::config::TransactionPoolOptions> { + match self { + $($enum::$variant(cmd) => cmd.transaction_pool()),* + } + } + + fn network_config( + &self, + chain_spec: &::std::boxed::Box, + is_dev: bool, + net_config_dir: &::std::path::PathBuf, + client_id: &str, + node_name: &str, + node_key: ::sc_service::config::NodeKeyConfig, + ) -> $crate::Result<::sc_service::config::NetworkConfiguration> { + match self { + $( + $enum::$variant(cmd) => cmd.network_config( + chain_spec, is_dev, net_config_dir, client_id, node_name, node_key + ) + ),* + } + } + + fn keystore_config(&self, base_path: &::std::path::PathBuf) + -> $crate::Result<::sc_service::config::KeystoreConfig> { + match self { + $($enum::$variant(cmd) => cmd.keystore_config(base_path)),* + } + } + + fn database_cache_size(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.database_cache_size()),* + } + } + + fn database_config( + &self, + base_path: &::std::path::PathBuf, + cache_size: usize, + ) -> $crate::Result<::sc_service::config::DatabaseConfig> { + match self { + $($enum::$variant(cmd) => cmd.database_config(base_path, cache_size)),* + } + } + + fn state_cache_size(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.state_cache_size()),* + } + } + + fn state_cache_child_ratio(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.state_cache_child_ratio()),* + } + } + + fn pruning(&self, is_dev: bool, role: &::sc_service::Role) + -> $crate::Result<::sc_service::config::PruningMode> { + match self { + $($enum::$variant(cmd) => cmd.pruning(is_dev, role)),* + } + } + + fn chain_id(&self, is_dev: bool) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.chain_id(is_dev)),* + } + } + + fn init(&self) -> $crate::Result<()> { + match self { + $($enum::$variant(cmd) => cmd.init::()),* + } + } + + fn node_name(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.node_name()),* + } + } + + fn wasm_method(&self) -> $crate::Result<::sc_service::config::WasmExecutionMethod> { + match self { + $($enum::$variant(cmd) => cmd.wasm_method()),* + } + } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - self.get_shared_params().init(version) + fn execution_strategies(&self, is_dev: bool) + -> $crate::Result<::sc_service::config::ExecutionStrategies> { + match self { + $($enum::$variant(cmd) => cmd.execution_strategies(is_dev)),* + } + } + + fn rpc_http(&self) -> $crate::Result<::std::option::Option<::std::net::SocketAddr>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_http()),* + } + } + + fn rpc_ws(&self) -> $crate::Result<::std::option::Option<::std::net::SocketAddr>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_ws()),* + } + } + + fn rpc_ws_max_connections(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.rpc_ws_max_connections()),* + } + } + + fn rpc_cors(&self, is_dev: bool) + -> $crate::Result<::std::option::Option<::std::vec::Vec>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_cors(is_dev)),* + } + } + + fn prometheus_config(&self) + -> $crate::Result<::std::option::Option<::sc_service::config::PrometheusConfig>> { + match self { + $($enum::$variant(cmd) => cmd.prometheus_config()),* + } + } + + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> $crate::Result<::std::option::Option<::sc_service::config::TelemetryEndpoints>> { + match self { + $($enum::$variant(cmd) => cmd.telemetry_endpoints(chain_spec)),* + } + } + + fn telemetry_external_transport(&self) + -> $crate::Result<::std::option::Option<::sc_service::config::ExtTransport>> { + match self { + $($enum::$variant(cmd) => cmd.telemetry_external_transport()),* + } + } + + fn default_heap_pages(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.default_heap_pages()),* + } + } + + fn offchain_worker(&self, role: &::sc_service::Role) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.offchain_worker(role)),* + } + } + + fn force_authoring(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.force_authoring()),* + } + } + + fn disable_grandpa(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.disable_grandpa()),* + } + } + + fn dev_key_seed(&self, is_dev: bool) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.dev_key_seed(is_dev)),* + } + } + + fn tracing_targets(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.tracing_targets()),* + } + } + + fn tracing_receiver(&self) -> $crate::Result<::sc_service::TracingReceiver> { + match self { + $($enum::$variant(cmd) => cmd.tracing_receiver()),* + } + } + + fn node_key(&self, net_config_dir: &::std::path::PathBuf) + -> $crate::Result<::sc_service::config::NodeKeyConfig> { + match self { + $($enum::$variant(cmd) => cmd.node_key(net_config_dir)),* + } + } + + fn max_runtime_instances(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.max_runtime_instances()),* + } + } + + fn log_filters(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.log_filters()),* + } + } + } } } + +substrate_cli_subcommands!( + Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain +); diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs index e12a50bf24f174ccfcc868fa4099bdf11fb45292..845423695eae8ee9b2229e28a8e499579a288f0b 100644 --- a/client/cli/src/commands/purge_chain_cmd.rs +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -14,15 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{config::DatabaseConfig, Configuration}; use std::fmt::Debug; -use std::io::{Write, self}; use std::fs; +use std::io::{self, Write}; use structopt::StructOpt; -use sc_service::{ Configuration, ChainSpec, config::{DatabaseConfig} }; - -use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; /// The `purge-chain` command used to remove the whole chain. #[derive(Debug, StructOpt, Clone)] @@ -38,11 +37,8 @@ pub struct PurgeChainCmd { impl PurgeChainCmd { /// Run the purge command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> { - let db_path = match config.expect_database() { + pub fn run(&self, config: Configuration) -> error::Result<()> { + let db_path = match &config.database { DatabaseConfig::Path { path, .. } => path, _ => { eprintln!("Cannot purge custom database implementation"); @@ -76,22 +72,13 @@ impl PurgeChainCmd { eprintln!("{:?} did not exist.", &db_path); Ok(()) }, - Err(err) => Result::Err(err.into()) + Err(err) => Result::Err(err.into()), } } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - config.use_in_memory_keystore()?; - - Ok(()) +impl CliConfiguration for PurgeChainCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params } } diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs index 8eba199dff07ba4b8f518e69842a4053ed0e8382..f7629ff2f6357e71ed417b0bad6d6ad1fc96851b 100644 --- a/client/cli/src/commands/revert_cmd.rs +++ b/client/cli/src/commands/revert_cmd.rs @@ -14,16 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::{BlockNumber, PruningParams, SharedParams}; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, Roles, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::params::{BlockNumber, SharedParams, PruningParams}; /// The `revert` command used revert the chain to a previous state. #[derive(Debug, StructOpt, Clone)] @@ -43,11 +40,7 @@ pub struct RevertCmd { impl RevertCmd { /// Run the revert command - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> + pub fn run(&self, config: Configuration, builder: B) -> error::Result<()> where B: FnOnce(Configuration) -> Result, BC: ServiceBuilderCommand + Unpin, @@ -60,20 +53,14 @@ impl RevertCmd { Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.pruning_params.update_config(&mut config, Roles::FULL, true)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for RevertCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) } } diff --git a/client/cli/src/commands/runcmd.rs b/client/cli/src/commands/runcmd.rs index c8af14359ce71bd429ab11f0d2f541c56e7e86d3..b3ce6ce6d119248a0b937c501328fa68e2fa8a6a 100644 --- a/client/cli/src/commands/runcmd.rs +++ b/client/cli/src/commands/runcmd.rs @@ -14,33 +14,21 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::path::PathBuf; -use std::net::SocketAddr; -use std::fs; -use log::info; -use structopt::{StructOpt, clap::arg_enum}; -use names::{Generator, Name}; +use crate::error::{Error, Result}; +use crate::params::ImportParams; +use crate::params::KeystoreParams; +use crate::params::NetworkParams; +use crate::params::SharedParams; +use crate::params::TransactionPoolParams; +use crate::CliConfiguration; use regex::Regex; -use chrono::prelude::*; use sc_service::{ - AbstractService, Configuration, ChainSpec, Roles, - config::{KeystoreConfig, PrometheusConfig}, + config::{MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions}, + ChainSpec, Role, }; use sc_telemetry::TelemetryEndpoints; - -use crate::VersionInfo; -use crate::error; -use crate::params::ImportParams; -use crate::params::SharedParams; -use crate::params::NetworkConfigurationParams; -use crate::params::TransactionPoolParams; -use crate::runtime::run_service_until_exit; - -/// The maximum number of characters for a node name. -const NODE_NAME_MAX_LENGTH: usize = 32; - -/// default sub directory for the key store -const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; +use std::net::SocketAddr; +use structopt::{clap::arg_enum, StructOpt}; arg_enum! { /// Whether off-chain workers are enabled. @@ -78,9 +66,10 @@ pub struct RunCmd { /// available to relay to private nodes. #[structopt( long = "sentry", - conflicts_with_all = &[ "validator", "light" ] + conflicts_with_all = &[ "validator", "light" ], + parse(try_from_str) )] - pub sentry: bool, + pub sentry: Vec, /// Disable GRANDPA voter when running in validator mode, otherwise disable the GRANDPA observer. #[structopt(long = "no-grandpa")] @@ -140,7 +129,7 @@ pub struct RunCmd { /// /// A comma-separated list of origins (protocol://domain or special `null` /// value). Value of `all` will disable origin validation. Default is to - /// allow localhost and https://polkadot.js.org origins. When running in + /// allow localhost and https://polkadot.js.org origins. When running in /// --dev mode the default is to allow all origins. #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] pub rpc_cors: Option, @@ -171,8 +160,8 @@ pub struct RunCmd { /// /// This flag can be passed multiple times as a means to specify multiple /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting - /// the least verbosity. If no verbosity level is specified the default is - /// 0. + /// the least verbosity. + /// Expected format is 'URL VERBOSITY', e.g. `--telemetry-url 'wss://foo/bar 0'`. #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] pub telemetry_endpoints: Vec<(String, u8)>, @@ -198,7 +187,7 @@ pub struct RunCmd { #[allow(missing_docs)] #[structopt(flatten)] - pub network_config: NetworkConfigurationParams, + pub network_params: NetworkParams, #[allow(missing_docs)] #[structopt(flatten)] @@ -240,38 +229,23 @@ pub struct RunCmd { #[structopt(long = "force-authoring")] pub force_authoring: bool, - /// Specify custom keystore path. - #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] - pub keystore_path: Option, - - /// Use interactive shell for entering the password used by the keystore. - #[structopt( - long = "password-interactive", - conflicts_with_all = &[ "password", "password-filename" ] - )] - pub password_interactive: bool, - - /// Password used by the keystore. - #[structopt( - long = "password", - conflicts_with_all = &[ "password-interactive", "password-filename" ] - )] - pub password: Option, - - /// File that contains the password used by the keystore. - #[structopt( - long = "password-filename", - value_name = "PATH", - parse(from_os_str), - conflicts_with_all = &[ "password-interactive", "password" ] - )] - pub password_filename: Option, + #[allow(missing_docs)] + #[structopt(flatten)] + pub keystore_params: KeystoreParams, /// The size of the instances cache for each runtime. /// /// The default value is 8 and the values higher than 256 are ignored. - #[structopt(long = "max-runtime-instances", default_value = "8")] - pub max_runtime_instances: usize, + #[structopt(long)] + pub max_runtime_instances: Option, + + /// Specify a list of sentry node public addresses. + #[structopt( + long = "sentry-nodes", + value_name = "ADDR", + conflicts_with_all = &[ "sentry" ] + )] + pub sentry_nodes: Vec, } impl RunCmd { @@ -279,220 +253,203 @@ impl RunCmd { pub fn get_keyring(&self) -> Option { use sp_keyring::Sr25519Keyring::*; - if self.alice { Some(Alice) } - else if self.bob { Some(Bob) } - else if self.charlie { Some(Charlie) } - else if self.dave { Some(Dave) } - else if self.eve { Some(Eve) } - else if self.ferdie { Some(Ferdie) } - else if self.one { Some(One) } - else if self.two { Some(Two) } - else { None } - } - - /// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo`. - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> - where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - - let password = if self.password_interactive { - #[cfg(not(target_os = "unknown"))] - { - Some(input_keystore_password()?.into()) - } - #[cfg(target_os = "unknown")] - None - } else if let Some(ref file) = self.password_filename { - Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) - } else if let Some(ref password) = self.password { - Some(password.clone().into()) + if self.alice { + Some(Alice) + } else if self.bob { + Some(Bob) + } else if self.charlie { + Some(Charlie) + } else if self.dave { + Some(Dave) + } else if self.eve { + Some(Eve) + } else if self.ferdie { + Some(Ferdie) + } else if self.one { + Some(One) + } else if self.two { + Some(Two) } else { None - }; + } + } +} - let path = self.keystore_path.clone().or( - config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) - ); +impl CliConfiguration for RunCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - config.keystore = KeystoreConfig::Path { - path: path.ok_or_else(|| "No `base_path` provided to create keystore path!".to_string())?, - password, - }; + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } - let keyring = self.get_keyring(); - let is_dev = self.shared_params.dev; - let is_light = self.light; - let is_authority = (self.validator || self.sentry || is_dev || keyring.is_some()) - && !is_light; - let role = - if is_light { - sc_service::Roles::LIGHT - } else if is_authority { - sc_service::Roles::AUTHORITY - } else { - sc_service::Roles::FULL - }; + fn network_params(&self) -> Option<&NetworkParams> { + Some(&self.network_params) + } - self.import_params.update_config(&mut config, role, is_dev)?; + fn keystore_params(&self) -> Option<&KeystoreParams> { + Some(&self.keystore_params) + } - config.name = match (self.name.as_ref(), keyring) { + fn node_name(&self) -> Result { + let name: String = match (self.name.as_ref(), self.get_keyring()) { (Some(name), _) => name.to_string(), (_, Some(keyring)) => keyring.to_string(), - (None, None) => generate_node_name(), + (None, None) => crate::generate_node_name(), }; - if let Err(msg) = is_node_name_valid(&config.name) { - return Err(error::Error::Input( - format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", - config.name, - msg, - ) + + is_node_name_valid(&name).map_err(|msg| { + Error::Input(format!( + "Invalid node name '{}'. Reason: {}. If unsure, use none.", + name, msg )); - } + })?; - // set sentry mode (i.e. act as an authority but **never** actively participate) - config.sentry_mode = self.sentry; + Ok(name) + } - config.offchain_worker = match (&self.offchain_worker, role) { - (OffchainWorkerEnabled::WhenValidating, sc_service::Roles::AUTHORITY) => true, - (OffchainWorkerEnabled::Always, _) => true, - (OffchainWorkerEnabled::Never, _) => false, - (OffchainWorkerEnabled::WhenValidating, _) => false, - }; + fn dev_key_seed(&self, is_dev: bool) -> Result> { + Ok(self.get_keyring().map(|a| format!("//{}", a)).or_else(|| { + if is_dev && !self.light { + Some("//Alice".into()) + } else { + None + } + })) + } - config.roles = role; - config.disable_grandpa = self.no_grandpa; - - let client_id = config.client_id(); - let network_path = config - .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a basepath"); - self.network_config.update_config( - &mut config, - network_path, - client_id, - is_dev, - )?; - - self.pool_config.update_config(&mut config)?; - - config.dev_key_seed = keyring - .map(|a| format!("//{}", a)).or_else(|| { - if is_dev && !is_light { - Some("//Alice".into()) - } else { - None - } - }); + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> Result> { + Ok(if self.no_telemetry { + None + } else if !self.telemetry_endpoints.is_empty() { + Some( + TelemetryEndpoints::new(self.telemetry_endpoints.clone()) + .map_err(|e| e.to_string())?, + ) + } else { + chain_spec.telemetry_endpoints().clone() + }) + } - if config.rpc_http.is_none() || self.rpc_port.is_some() { - let rpc_interface: &str = interface_str(self.rpc_external, self.unsafe_rpc_external, self.validator)?; - config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), self.rpc_port)?); - } - if config.rpc_ws.is_none() || self.ws_port.is_some() { - let ws_interface: &str = interface_str(self.ws_external, self.unsafe_ws_external, self.validator)?; - config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), self.ws_port)?); - } + fn role(&self, is_dev: bool) -> Result { + let keyring = self.get_keyring(); + let is_light = self.light; + let is_authority = (self.validator || is_dev || keyring.is_some()) && !is_light; - config.rpc_ws_max_connections = self.ws_max_connections; - config.rpc_cors = self.rpc_cors.clone().unwrap_or_else(|| if is_dev { - log::warn!("Running in --dev mode, RPC CORS has been disabled."); - Cors::All + Ok(if is_light { + sc_service::Role::Light + } else if is_authority { + sc_service::Role::Authority { + sentry_nodes: self.sentry_nodes.clone(), + } + } else if !self.sentry.is_empty() { + sc_service::Role::Sentry { + validators: self.sentry.clone(), + } } else { - Cors::List(vec![ - "http://localhost:*".into(), - "http://127.0.0.1:*".into(), - "https://localhost:*".into(), - "https://127.0.0.1:*".into(), - "https://polkadot.js.org".into(), - ]) - }).into(); - - // Override telemetry - if self.no_telemetry { - config.telemetry_endpoints = None; - } else if !self.telemetry_endpoints.is_empty() { - config.telemetry_endpoints = Some( - TelemetryEndpoints::new(self.telemetry_endpoints.clone()) - ); - } + sc_service::Role::Full + }) + } - // Override prometheus + fn force_authoring(&self) -> Result { + // Imply forced authoring on --dev + Ok(self.shared_params.dev || self.force_authoring) + } + + fn prometheus_config(&self) -> Result> { if self.no_prometheus { - config.prometheus_config = None; - } else if config.prometheus_config.is_none() { - let prometheus_interface: &str = if self.prometheus_external { "0.0.0.0" } else { "127.0.0.1" }; - config.prometheus_config = Some(PrometheusConfig::new_with_default_registry( - parse_address(&format!("{}:{}", prometheus_interface, 9615), self.prometheus_port)?, - )); + Ok(None) + } else { + let prometheus_interface: &str = if self.prometheus_external { + "0.0.0.0" + } else { + "127.0.0.1" + }; + + Ok(Some(PrometheusConfig::new_with_default_registry( + parse_address( + &format!("{}:{}", prometheus_interface, 9615), + self.prometheus_port, + )?, + ))) } + } - config.tracing_targets = self.import_params.tracing_targets.clone().into(); - config.tracing_receiver = self.import_params.tracing_receiver.clone().into(); + fn disable_grandpa(&self) -> Result { + Ok(self.no_grandpa) + } - // Imply forced authoring on --dev - config.force_authoring = self.shared_params.dev || self.force_authoring; - - config.max_runtime_instances = self.max_runtime_instances.min(256); - - Ok(()) - } - - /// Run the command that runs the node. - pub fn run( - self, - config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, - ) -> error::Result<()> - where - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, - { - info!("{}", version.name); - info!(" version {}", config.full_version()); - info!(" by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); - info!("Chain specification: {}", config.expect_chain_spec().name()); - info!("Node name: {}", config.name); - info!("Roles: {}", config.display_role()); - - match config.roles { - Roles::LIGHT => run_service_until_exit( - config, - new_light, - ), - _ => run_service_until_exit( - config, - new_full, - ), - } + fn rpc_ws_max_connections(&self) -> Result> { + Ok(self.ws_max_connections) } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - self.shared_params.init(version) + fn rpc_cors(&self, is_dev: bool) -> Result>> { + Ok(self + .rpc_cors + .clone() + .unwrap_or_else(|| { + if is_dev { + log::warn!("Running in --dev mode, RPC CORS has been disabled."); + Cors::All + } else { + Cors::List(vec![ + "http://localhost:*".into(), + "http://127.0.0.1:*".into(), + "https://localhost:*".into(), + "https://127.0.0.1:*".into(), + "https://polkadot.js.org".into(), + ]) + } + }) + .into()) + } + + fn rpc_http(&self) -> Result> { + let rpc_interface: &str = + interface_str(self.rpc_external, self.unsafe_rpc_external, self.validator)?; + + Ok(Some(parse_address( + &format!("{}:{}", rpc_interface, 9933), + self.rpc_port, + )?)) + } + + fn rpc_ws(&self) -> Result> { + let ws_interface: &str = + interface_str(self.ws_external, self.unsafe_ws_external, self.validator)?; + + Ok(Some(parse_address( + &format!("{}:{}", ws_interface, 9944), + self.ws_port, + )?)) + } + + fn offchain_worker(&self, role: &Role) -> Result { + Ok(match (&self.offchain_worker, role) { + (OffchainWorkerEnabled::WhenValidating, Role::Authority { .. }) => true, + (OffchainWorkerEnabled::Always, _) => true, + (OffchainWorkerEnabled::Never, _) => false, + (OffchainWorkerEnabled::WhenValidating, _) => false, + }) + } + + fn transaction_pool(&self) -> Result { + Ok(self.pool_config.transaction_pool()) + } + + fn max_runtime_instances(&self) -> Result> { + Ok(self.max_runtime_instances.map(|x| x.min(256))) } } /// Check whether a node name is considered as valid. -pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { +pub fn is_node_name_valid(_name: &str) -> std::result::Result<(), &str> { let name = _name.to_string(); - if name.chars().count() >= NODE_NAME_MAX_LENGTH { + if name.chars().count() >= crate::NODE_NAME_MAX_LENGTH { return Err("Node name too long"); } @@ -511,32 +468,10 @@ pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { Ok(()) } -#[cfg(not(target_os = "unknown"))] -fn input_keystore_password() -> Result { - rpassword::read_password_from_tty(Some("Keystore password: ")) - .map_err(|e| format!("{:?}", e)) -} - -fn generate_node_name() -> String { - let result = loop { - let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); - let count = node_name.chars().count(); - - if count < NODE_NAME_MAX_LENGTH { - break node_name - } - }; - - result -} - -fn parse_address( - address: &str, - port: Option, -) -> Result { - let mut address: SocketAddr = address.parse().map_err( - |_| format!("Invalid address: {}", address) - )?; +fn parse_address(address: &str, port: Option) -> std::result::Result { + let mut address: SocketAddr = address + .parse() + .map_err(|_| format!("Invalid address: {}", address))?; if let Some(port) = port { address.set_port(port); } @@ -548,16 +483,21 @@ fn interface_str( is_external: bool, is_unsafe_external: bool, is_validator: bool, -) -> Result<&'static str, error::Error> { +) -> Result<&'static str> { if is_external && is_validator { - return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ + return Err(Error::Input( + "--rpc-external and --ws-external options shouldn't be \ used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ - the risks. See the options description for more information.".to_owned())); + the risks. See the options description for more information." + .to_owned(), + )); } if is_external || is_unsafe_external { - log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ - available set of RPC methods."); + log::warn!( + "It isn't safe to expose RPC publicly without a proxy server that filters \ + available set of RPC methods." + ); Ok("0.0.0.0") } else { @@ -565,16 +505,32 @@ fn interface_str( } } -/// Default to verbosity level 0, if none is provided. -fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { +#[derive(Debug)] +enum TelemetryParsingError { + MissingVerbosity, + VerbosityParsingError(std::num::ParseIntError), +} + +impl std::error::Error for TelemetryParsingError {} + +impl std::fmt::Display for TelemetryParsingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &*self { + TelemetryParsingError::MissingVerbosity => write!(f, "Verbosity level missing"), + TelemetryParsingError::VerbosityParsingError(e) => write!(f, "{}", e), + } + } +} + +fn parse_telemetry_endpoints(s: &str) -> std::result::Result<(String, u8), TelemetryParsingError> { let pos = s.find(' '); match pos { - None => { - Ok((s.to_owned(), 0)) - }, + None => Err(TelemetryParsingError::MissingVerbosity), Some(pos_) => { - let verbosity = s[pos_ + 1..].parse()?; - let url = s[..pos_].parse()?; + let url = s[..pos_].to_string(); + let verbosity = s[pos_ + 1..] + .parse() + .map_err(TelemetryParsingError::VerbosityParsingError)?; Ok((url, verbosity)) } } @@ -602,7 +558,7 @@ impl From for Option> { } /// Parse cors origins. -fn parse_cors(s: &str) -> Result> { +fn parse_cors(s: &str) -> std::result::Result> { let mut is_all = false; let mut origins = Vec::new(); for part in s.split(',') { @@ -610,29 +566,21 @@ fn parse_cors(s: &str) -> Result> { "all" | "*" => { is_all = true; break; - }, + } other => origins.push(other.to_owned()), } } - Ok(if is_all { Cors::All } else { Cors::List(origins) }) + Ok(if is_all { + Cors::All + } else { + Cors::List(origins) + }) } #[cfg(test)] mod tests { use super::*; - use sc_service::{GenericChainSpec, config::DatabaseConfig}; - - const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { - name: "node-test", - version: "0.1.0", - commit: "some_commit", - executable_name: "node-test", - description: "description", - author: "author", - support_url: "http://example.org", - copyright_start_year: 2020, - }; #[test] fn tests_node_name_good() { @@ -648,93 +596,4 @@ mod tests { assert!(is_node_name_valid("www.visit.me").is_err()); assert!(is_node_name_valid("email@domain").is_err()); } - - #[test] - fn keystore_path_is_generated_correctly() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - Vec::new(), - None, - None, - None, - None::<()>, - ); - - for keystore_path in vec![None, Some("/keystore/path")] { - let args: Vec<&str> = vec![]; - let mut cli = RunCmd::from_iter(args); - cli.keystore_path = keystore_path.clone().map(PathBuf::from); - - let mut config = Configuration::default(); - config.config_dir = Some(PathBuf::from("/test/path")); - config.chain_spec = Some(Box::new(chain_spec.clone())); - let chain_spec = chain_spec.clone(); - cli.update_config(&mut config, move |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - let expected_path = match keystore_path { - Some(path) => PathBuf::from(path), - None => PathBuf::from("/test/path/chains/test-id/keystore"), - }; - - assert_eq!(expected_path, config.keystore.path().unwrap().to_owned()); - } - } - - #[test] - fn ensure_load_spec_provide_defaults() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - vec!["boo".to_string()], - Some(TelemetryEndpoints::new(vec![("foo".to_string(), 42)])), - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::from_version(TEST_VERSION_INFO); - cli.update_config(&mut config, |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - assert!(config.chain_spec.is_some()); - assert!(!config.network.boot_nodes.is_empty()); - assert!(config.telemetry_endpoints.is_some()); - } - - #[test] - fn ensure_update_config_for_running_node_provides_defaults() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - vec![], - None, - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::from_version(TEST_VERSION_INFO); - cli.init(&TEST_VERSION_INFO).unwrap(); - cli.update_config(&mut config, |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - assert!(config.config_dir.is_some()); - assert!(config.database.is_some()); - if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { - assert!(cache_size.is_some()); - } else { - panic!("invalid config.database variant"); - } - assert!(!config.name.is_empty()); - assert!(config.network.config_path.is_some()); - assert!(!config.network.listen_addresses.is_empty()); - } } diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs new file mode 100644 index 0000000000000000000000000000000000000000..6c4cc0e7103f9147711a609dd14dd9f64f9d98c5 --- /dev/null +++ b/client/cli/src/config.rs @@ -0,0 +1,462 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Configuration trait for a CLI based on substrate + +use crate::error::Result; +use crate::{ + init_logger, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams, + PruningParams, SharedParams, SubstrateCli, +}; +use app_dirs::{AppDataType, AppInfo}; +use names::{Generator, Name}; +use sc_service::config::{ + Configuration, DatabaseConfig, ExecutionStrategies, ExtTransport, KeystoreConfig, + NetworkConfiguration, NodeKeyConfig, PrometheusConfig, PruningMode, Role, TelemetryEndpoints, + TransactionPoolOptions, WasmExecutionMethod, +}; +use sc_service::{ChainSpec, TracingReceiver}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +/// The maximum number of characters for a node name. +pub(crate) const NODE_NAME_MAX_LENGTH: usize = 32; + +/// default sub directory to store network config +pub(crate) const DEFAULT_NETWORK_CONFIG_PATH: &'static str = "network"; + +/// A trait that allows converting an object to a Configuration +pub trait CliConfiguration: Sized { + /// Get the SharedParams for this object + fn shared_params(&self) -> &SharedParams; + + /// Get the ImportParams for this object + fn import_params(&self) -> Option<&ImportParams> { + None + } + + /// Get the PruningParams for this object + fn pruning_params(&self) -> Option<&PruningParams> { + self.import_params().map(|x| &x.pruning_params) + } + + /// Get the KeystoreParams for this object + fn keystore_params(&self) -> Option<&KeystoreParams> { + None + } + + /// Get the NetworkParams for this object + fn network_params(&self) -> Option<&NetworkParams> { + None + } + + /// Get the NodeKeyParams for this object + fn node_key_params(&self) -> Option<&NodeKeyParams> { + self.network_params() + .map(|x| &x.node_key_params) + } + + /// Get the base path of the configuration (if any) + /// + /// By default this is retrieved from `SharedParams`. + fn base_path(&self) -> Result> { + Ok(self.shared_params().base_path()) + } + + /// Returns `true` if the node is for development or not + /// + /// By default this is retrieved from `SharedParams`. + fn is_dev(&self) -> Result { + Ok(self.shared_params().is_dev()) + } + + /// Gets the role + /// + /// By default this is `Role::Full`. + fn role(&self, _is_dev: bool) -> Result { + Ok(Role::Full) + } + + /// Get the transaction pool options + /// + /// By default this is `TransactionPoolOptions::default()`. + fn transaction_pool(&self) -> Result { + Ok(Default::default()) + } + + /// Get the network configuration + /// + /// By default this is retrieved from `NetworkParams` if it is available otherwise it creates + /// a default `NetworkConfiguration` based on `node_name`, `client_id`, `node_key` and + /// `net_config_dir`. + fn network_config( + &self, + chain_spec: &Box, + is_dev: bool, + net_config_dir: &PathBuf, + client_id: &str, + node_name: &str, + node_key: NodeKeyConfig, + ) -> Result { + Ok(if let Some(network_params) = self.network_params() { + network_params.network_config( + chain_spec, + is_dev, + net_config_dir, + client_id, + node_name, + node_key, + ) + } else { + NetworkConfiguration::new( + node_name, + client_id, + node_key, + net_config_dir, + ) + }) + } + + /// Get the keystore configuration. + /// + /// Bu default this is retrieved from `KeystoreParams` if it is available. Otherwise it uses + /// `KeystoreConfig::InMemory`. + fn keystore_config(&self, base_path: &PathBuf) -> Result { + self.keystore_params() + .map(|x| x.keystore_config(base_path)) + .unwrap_or(Ok(KeystoreConfig::InMemory)) + } + + /// Get the database cache size. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its `None`. + fn database_cache_size(&self) -> Result> { + Ok(self.import_params() + .map(|x| x.database_cache_size()) + .unwrap_or(Default::default())) + } + + /// Get the database configuration. + /// + /// By default this is retrieved from `SharedParams` + fn database_config(&self, base_path: &PathBuf, cache_size: usize) -> Result { + Ok(self.shared_params().database_config(base_path, cache_size)) + } + + /// Get the state cache size. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its `0`. + fn state_cache_size(&self) -> Result { + Ok(self.import_params() + .map(|x| x.state_cache_size()) + .unwrap_or(Default::default())) + } + + /// Get the state cache child ratio (if any). + /// + /// By default this is `None`. + fn state_cache_child_ratio(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the pruning mode. + /// + /// By default this is retrieved from `PruningMode` if it is available. Otherwise its + /// `PruningMode::default()`. + fn pruning(&self, is_dev: bool, role: &Role) -> Result { + self.pruning_params() + .map(|x| x.pruning(is_dev, role)) + .unwrap_or(Ok(Default::default())) + } + + /// Get the chain ID (string). + /// + /// By default this is retrieved from `SharedParams`. + fn chain_id(&self, is_dev: bool) -> Result { + Ok(self.shared_params().chain_id(is_dev)) + } + + /// Get the name of the node. + /// + /// By default a random name is generated. + fn node_name(&self) -> Result { + Ok(generate_node_name()) + } + + /// Get the WASM execution method. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `WasmExecutionMethod::default()`. + fn wasm_method(&self) -> Result { + Ok(self.import_params() + .map(|x| x.wasm_method()) + .unwrap_or(Default::default())) + } + + /// Get the execution strategies. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `ExecutionStrategies::default()`. + fn execution_strategies(&self, is_dev: bool) -> Result { + Ok(self.import_params() + .map(|x| x.execution_strategies(is_dev)) + .unwrap_or(Default::default())) + } + + /// Get the RPC HTTP address (`None` if disabled). + /// + /// By default this is `None`. + fn rpc_http(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the RPC websocket address (`None` if disabled). + /// + /// By default this is `None`. + fn rpc_ws(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the RPC websockets maximum connections (`None` if unlimited). + /// + /// By default this is `None`. + fn rpc_ws_max_connections(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the RPC cors (`None` if disabled) + /// + /// By default this is `None`. + fn rpc_cors(&self, _is_dev: bool) -> Result>> { + Ok(Some(Vec::new())) + } + + /// Get the prometheus configuration (`None` if disabled) + /// + /// By default this is `None`. + fn prometheus_config(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the telemetry endpoints (if any) + /// + /// By default this is retrieved from the chain spec loaded by `load_spec`. + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> Result> { + Ok(chain_spec.telemetry_endpoints().clone()) + } + + /// Get the telemetry external transport + /// + /// By default this is `None`. + fn telemetry_external_transport(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the default value for heap pages + /// + /// By default this is `None`. + fn default_heap_pages(&self) -> Result> { + Ok(Default::default()) + } + + /// Returns `Ok(true)` if offchain worker should be used + /// + /// By default this is `false`. + fn offchain_worker(&self, _role: &Role) -> Result { + Ok(Default::default()) + } + + /// Returns `Ok(true)` if authoring should be forced + /// + /// By default this is `false`. + fn force_authoring(&self) -> Result { + Ok(Default::default()) + } + + /// Returns `Ok(true)` if grandpa should be disabled + /// + /// By default this is `false`. + fn disable_grandpa(&self) -> Result { + Ok(Default::default()) + } + + /// Get the development key seed from the current object + /// + /// By default this is `None`. + fn dev_key_seed(&self, _is_dev: bool) -> Result> { + Ok(Default::default()) + } + + /// Get the tracing targets from the current object (if any) + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `None`. + fn tracing_targets(&self) -> Result> { + Ok(self.import_params() + .map(|x| x.tracing_targets()) + .unwrap_or(Default::default())) + } + + /// Get the TracingReceiver value from the current object + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `TracingReceiver::default()`. + fn tracing_receiver(&self) -> Result { + Ok(self.import_params() + .map(|x| x.tracing_receiver()) + .unwrap_or(Default::default())) + } + + /// Get the node key from the current object + /// + /// By default this is retrieved from `NodeKeyParams` if it is available. Otherwise its + /// `NodeKeyConfig::default()`. + fn node_key(&self, net_config_dir: &PathBuf) -> Result { + self.node_key_params() + .map(|x| x.node_key(net_config_dir)) + .unwrap_or(Ok(Default::default())) + } + + /// Get maximum runtime instances + /// + /// By default this is `None`. + fn max_runtime_instances(&self) -> Result> { + Ok(Default::default()) + } + + /// Activate or not the automatic announcing of blocks after import + /// + /// By default this is `false`. + fn announce_block(&self) -> Result { + Ok(true) + } + + /// Create a Configuration object from the current object + fn create_configuration( + &self, + cli: &C, + task_executor: Arc + Send>>) + Send + Sync>, + ) -> Result { + let is_dev = self.is_dev()?; + let chain_id = self.chain_id(is_dev)?; + let chain_spec = cli.load_spec(chain_id.as_str())?; + let config_dir = self + .base_path()? + .unwrap_or_else(|| { + app_dirs::get_app_root( + AppDataType::UserData, + &AppInfo { + name: C::executable_name(), + author: C::author(), + }, + ) + .expect("app directories exist on all supported platforms; qed") + }) + .join("chains") + .join(chain_spec.id()); + let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH); + let client_id = C::client_id(); + let database_cache_size = self.database_cache_size()?.unwrap_or(128); + let node_key = self.node_key(&net_config_dir)?; + let role = self.role(is_dev)?; + let max_runtime_instances = self.max_runtime_instances()?.unwrap_or(8); + + Ok(Configuration { + impl_name: C::impl_name(), + impl_version: C::impl_version(), + task_executor, + transaction_pool: self.transaction_pool()?, + network: self.network_config( + &chain_spec, + is_dev, + &net_config_dir, + client_id.as_str(), + self.node_name()?.as_str(), + node_key, + )?, + keystore: self.keystore_config(&config_dir)?, + database: self.database_config(&config_dir, database_cache_size)?, + state_cache_size: self.state_cache_size()?, + state_cache_child_ratio: self.state_cache_child_ratio()?, + pruning: self.pruning(is_dev, &role)?, + wasm_method: self.wasm_method()?, + execution_strategies: self.execution_strategies(is_dev)?, + rpc_http: self.rpc_http()?, + rpc_ws: self.rpc_ws()?, + rpc_ws_max_connections: self.rpc_ws_max_connections()?, + rpc_cors: self.rpc_cors(is_dev)?, + prometheus_config: self.prometheus_config()?, + telemetry_endpoints: self.telemetry_endpoints(&chain_spec)?, + telemetry_external_transport: self.telemetry_external_transport()?, + default_heap_pages: self.default_heap_pages()?, + offchain_worker: self.offchain_worker(&role)?, + force_authoring: self.force_authoring()?, + disable_grandpa: self.disable_grandpa()?, + dev_key_seed: self.dev_key_seed(is_dev)?, + tracing_targets: self.tracing_targets()?, + tracing_receiver: self.tracing_receiver()?, + chain_spec, + max_runtime_instances, + announce_block: self.announce_block()?, + role, + }) + } + + /// Get the filters for the logging. + /// + /// By default this is retrieved from `SharedParams`. + fn log_filters(&self) -> Result> { + Ok(self.shared_params().log_filters()) + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + fn init(&self) -> Result<()> { + let logger_pattern = self.log_filters()?.unwrap_or_default(); + + sp_panic_handler::set(C::support_url(), C::impl_version()); + + fdlimit::raise_fd_limit(); + init_logger(logger_pattern.as_str()); + + Ok(()) + } +} + +/// Generate a valid random name for the node +pub fn generate_node_name() -> String { + loop { + let node_name = Generator::with_naming(Name::Numbered) + .next() + .expect("RNG is available on all supported platforms; qed"); + let count = node_name.chars().count(); + + if count < NODE_NAME_MAX_LENGTH { + return node_name; + } + }; +} diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index e28edebd60def53ba34497d416a9bea7919ef983..25b71059b17e62a397bde14bccee91fe95fb62cd 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -19,137 +19,175 @@ #![warn(missing_docs)] #![warn(unused_extern_crates)] -mod params; mod arg_enums; -mod error; -mod runtime; mod commands; +mod config; +mod error; +mod params; +mod runner; -pub use sc_service::config::VersionInfo; - -use std::io::Write; - -use regex::Regex; -use structopt::{StructOpt, clap::{self, AppSettings}}; -pub use structopt; -pub use params::*; -pub use commands::*; pub use arg_enums::*; +pub use commands::*; +pub use config::*; pub use error::*; -use log::info; use lazy_static::lazy_static; -pub use crate::runtime::{run_until_exit, run_service_until_exit}; +use log::info; +pub use params::*; +use regex::Regex; +pub use runner::*; +use sc_service::{ChainSpec, Configuration}; +use std::future::Future; +use std::io::Write; +use std::pin::Pin; +use std::sync::Arc; +pub use structopt; +use structopt::{ + clap::{self, AppSettings}, + StructOpt, +}; -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// Substrate client CLI /// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. +/// This trait needs to be defined on the root structopt of the application. It will provide the +/// implementation name, version, executable name, description, author, support_url, copyright start +/// year and most importantly: how to load the chain spec. /// -/// Gets the struct from the command line arguments. Print the -/// error message and quit the program in case of failure. -pub fn from_args(version: &VersionInfo) -> T -where - T: StructOpt + Sized, -{ - from_iter::(&mut std::env::args_os(), version) -} +/// StructOpt must not be in scope to use from_args (or the similar methods). This trait provides +/// its own implementation that will fill the necessary field based on the trait's functions. +pub trait SubstrateCli: Sized { + /// Implementation name. + fn impl_name() -> &'static str; + + /// Implementation version. + /// + /// By default this will look like this: 2.0.0-b950f731c-x86_64-linux-gnu where the hash is the + /// short commit hash of the commit of in the Git repository. + fn impl_version() -> &'static str; + + /// Executable file name. + fn executable_name() -> &'static str; + + /// Executable file description. + fn description() -> &'static str; + + /// Executable file author. + fn author() -> &'static str; + + /// Support URL. + fn support_url() -> &'static str; + + /// Copyright starting year (x-current year) + fn copyright_start_year() -> i32; + + /// Chain spec factory + fn load_spec(&self, id: &str) -> std::result::Result, String>; + + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, tt also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from the command line arguments. Print the + /// error message and quit the program in case of failure. + fn from_args() -> Self + where + Self: StructOpt + Sized, + { + ::from_iter(&mut std::env::args_os()) + } -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. -/// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. -/// -/// Gets the struct from any iterator such as a `Vec` of your making. -/// Print the error message and quit the program in case of failure. -pub fn from_iter(iter: I, version: &VersionInfo) -> T -where - T: StructOpt + Sized, - I: IntoIterator, - I::Item: Into + Clone, -{ - let app = T::clap(); - - let mut full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit - ); - full_version.push_str("\n"); - - let app = app - .name(version.executable_name) - .author(version.author) - .about(version.description) - .version(full_version.as_str()) - .settings(&[ - AppSettings::GlobalVersion, - AppSettings::ArgsNegateSubcommands, - AppSettings::SubcommandsNegateReqs, - ]); - - T::from_clap(&app.get_matches_from(iter)) -} + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, it also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from any iterator such as a `Vec` of your making. + /// Print the error message and quit the program in case of failure. + fn from_iter(iter: I) -> Self + where + Self: StructOpt + Sized, + I: IntoIterator, + I::Item: Into + Clone, + { + let app = ::clap(); + + let mut full_version = Self::impl_version().to_string(); + full_version.push_str("\n"); + + let app = app + .name(Self::executable_name()) + .author(Self::author()) + .about(Self::description()) + .version(full_version.as_str()) + .settings(&[ + AppSettings::GlobalVersion, + AppSettings::ArgsNegateSubcommands, + AppSettings::SubcommandsNegateReqs, + ]); + + ::from_clap(&app.get_matches_from(iter)) + } -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. -/// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. -/// -/// Gets the struct from any iterator such as a `Vec` of your making. -/// Print the error message and quit the program in case of failure. -/// -/// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are -/// used. It will return a [`clap::Error`], where the [`kind`] is a -/// [`ErrorKind::HelpDisplayed`] or [`ErrorKind::VersionDisplayed`] respectively. You must call -/// [`Error::exit`] or perform a [`std::process::exit`]. -pub fn try_from_iter(iter: I, version: &VersionInfo) -> clap::Result -where - T: StructOpt + Sized, - I: IntoIterator, - I::Item: Into + Clone, -{ - let app = T::clap(); - - let mut full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit, - ); - full_version.push_str("\n"); - - let app = app - .name(version.executable_name) - .author(version.author) - .about(version.description) - .version(full_version.as_str()); - - let matches = app.get_matches_from_safe(iter)?; - - Ok(T::from_clap(&matches)) -} + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, it also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from any iterator such as a `Vec` of your making. + /// Print the error message and quit the program in case of failure. + /// + /// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are + /// used. It will return a [`clap::Error`], where the [`kind`] is a + /// [`ErrorKind::HelpDisplayed`] or [`ErrorKind::VersionDisplayed`] respectively. You must call + /// [`Error::exit`] or perform a [`std::process::exit`]. + fn try_from_iter(iter: I) -> clap::Result + where + Self: StructOpt + Sized, + I: IntoIterator, + I::Item: Into + Clone, + { + let app = ::clap(); + + let mut full_version = Self::impl_version().to_string(); + full_version.push_str("\n"); + + let app = app + .name(Self::executable_name()) + .author(Self::author()) + .about(Self::description()) + .version(full_version.as_str()); + + let matches = app.get_matches_from_safe(iter)?; + + Ok(::from_clap(&matches)) + } -/// Initialize substrate. This must be done only once. -/// -/// This method: -/// -/// 1. Set the panic handler -/// 2. Raise the FD limit -/// 3. Initialize the logger -pub fn init(logger_pattern: &str, version: &VersionInfo) -> error::Result<()> { - let full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit - ); - sp_panic_handler::set(version.support_url, &full_version); - - fdlimit::raise_fd_limit(); - init_logger(logger_pattern); + /// Returns the client ID: `{impl_name}/v{impl_version}` + fn client_id() -> String { + format!("{}/v{}", Self::impl_name(), Self::impl_version()) + } - Ok(()) + /// Only create a Configuration for the command provided in argument + fn create_configuration( + &self, + command: &T, + task_executor: Arc + Send>>) + Send + Sync>, + ) -> error::Result { + command.create_configuration(self, task_executor) + } + + /// Create a runner for the command provided in argument. This will create a Configuration and + /// a tokio runtime + fn create_runner(&self, command: &T) -> error::Result> { + command.init::()?; + Runner::new(self, command) + } } /// Initialize the logger @@ -177,17 +215,22 @@ pub fn init_logger(pattern: &str) { builder.format(move |buf, record| { let now = time::now(); let timestamp = - time::strftime("%Y-%m-%d %H:%M:%S", &now) - .expect("Error formatting log timestamp"); + time::strftime("%Y-%m-%d %H:%M:%S", &now).expect("Error formatting log timestamp"); let mut output = if log::max_level() <= log::LevelFilter::Info { - format!("{} {}", Colour::Black.bold().paint(timestamp), record.args()) + format!( + "{} {}", + Colour::Black.bold().paint(timestamp), + record.args(), + ) } else { let name = ::std::thread::current() .name() - .map_or_else(Default::default, |x| format!("{}", Colour::Blue.bold().paint(x))); - let millis = (now.tm_nsec as f32 / 1000000.0).round() as usize; - let timestamp = format!("{}.{:03}", timestamp, millis); + .map_or_else(Default::default, |x| { + format!("{}", Colour::Blue.bold().paint(x)) + }); + let millis = (now.tm_nsec as f32 / 1000000.0).floor() as usize; + let timestamp = format!("{}.{}", timestamp, millis); format!( "{} {} {} {} {}", Colour::Black.bold().paint(timestamp), @@ -211,7 +254,7 @@ pub fn init_logger(pattern: &str) { }); if builder.try_init().is_err() { - info!("Not registering Substrate logger, as there is already a global logger registered!"); + info!("💬 Not registering Substrate logger, as there is already a global logger registered!"); } } @@ -221,3 +264,21 @@ fn kill_color(s: &str) -> String { } RE.replace_all(s, "").to_string() } + +/// Reset the signal pipe (`SIGPIPE`) handler to the default one provided by the system. +/// This will end the program on `SIGPIPE` instead of panicking. +/// +/// This should be called before calling any cli method or printing any output. +pub fn reset_signal_pipe_handler() -> Result<()> { + #[cfg(target_family = "unix")] + { + use nix::sys::signal; + + unsafe { + signal::signal(signal::Signal::SIGPIPE, signal::SigHandler::SigDfl) + .map_err(|e| Error::Other(e.to_string()))?; + } + } + + Ok(()) +} diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index b647feeece30b8c34b74b0a33633f5380bfb21fd..08ca1c8f8ffe073951030feb965e29c616c67cd0 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -14,16 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use sc_service::{Configuration, config::DatabaseConfig}; - -use crate::error; use crate::arg_enums::{ - WasmExecutionMethod, TracingReceiver, ExecutionStrategy, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, + ExecutionStrategy, TracingReceiver, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, - DEFAULT_EXECUTION_SYNCING + DEFAULT_EXECUTION_SYNCING, }; use crate::params::PruningParams; +use crate::Result; +use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_service::{PruningMode, Role}; +use structopt::StructOpt; /// Parameters for block import. #[derive(Debug, StructOpt, Clone)] @@ -52,11 +52,11 @@ pub struct ImportParams { #[allow(missing_docs)] #[structopt(flatten)] - pub execution_strategies: ExecutionStrategies, + pub execution_strategies: ExecutionStrategiesParams, /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] - pub database_cache_size: u32, + #[structopt(long = "db-cache", value_name = "MiB")] + pub database_cache_size: Option, /// Specify the state cache size. #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] @@ -78,25 +78,31 @@ pub struct ImportParams { } impl ImportParams { - /// Put block import CLI params into `config` object. - pub fn update_config( - &self, - mut config: &mut Configuration, - role: sc_service::Roles, - is_dev: bool, - ) -> error::Result<()> { - use sc_client_api::execution_extensions::ExecutionStrategies; - - if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { - *cache_size = Some(self.database_cache_size); - } + /// Receiver to process tracing messages. + pub fn tracing_receiver(&self) -> sc_service::TracingReceiver { + self.tracing_receiver.clone().into() + } - config.state_cache_size = self.state_cache_size; + /// Comma separated list of targets for tracing. + pub fn tracing_targets(&self) -> Option { + self.tracing_targets.clone() + } - self.pruning_params.update_config(&mut config, role, self.unsafe_pruning)?; + /// Specify the state cache size. + pub fn state_cache_size(&self) -> usize { + self.state_cache_size + } - config.wasm_method = self.wasm_method.into(); + /// Get the WASM execution method from the parameters + pub fn wasm_method(&self) -> sc_service::config::WasmExecutionMethod { + self.wasm_method.into() + } + /// Get execution strategies for the parameters + pub fn execution_strategies( + &self, + is_dev: bool, + ) -> ExecutionStrategies { let exec = &self.execution_strategies; let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { exec.execution.unwrap_or(if strat == default && is_dev { @@ -106,7 +112,7 @@ impl ImportParams { }).into() }; - config.execution_strategies = ExecutionStrategies { + ExecutionStrategies { syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), block_construction: @@ -114,15 +120,23 @@ impl ImportParams { offchain_worker: exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), - }; + } + } - Ok(()) + /// Get the pruning mode from the parameters + pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> Result { + self.pruning_params.pruning(unsafe_pruning, role) + } + + /// Limit the memory the database cache can use. + pub fn database_cache_size(&self) -> Option { + self.database_cache_size } } /// Execution strategies parameters. #[derive(Debug, StructOpt, Clone)] -pub struct ExecutionStrategies { +pub struct ExecutionStrategiesParams { /// The means of execution used when calling into the runtime while syncing blocks. #[structopt( long = "execution-syncing", diff --git a/client/cli/src/params/keystore_params.rs b/client/cli/src/params/keystore_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..c6131c2f64941afa507931e5e090a4389a1e61eb --- /dev/null +++ b/client/cli/src/params/keystore_params.rs @@ -0,0 +1,92 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::error::Result; +use sc_service::config::KeystoreConfig; +use std::fs; +use std::path::PathBuf; +use structopt::StructOpt; + +/// default sub directory for the key store +const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore"; + +/// Parameters of the keystore +#[derive(Debug, StructOpt, Clone)] +pub struct KeystoreParams { + /// Specify custom keystore path. + #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] + pub keystore_path: Option, + + /// Use interactive shell for entering the password used by the keystore. + #[structopt( + long = "password-interactive", + conflicts_with_all = &[ "password", "password-filename" ] + )] + pub password_interactive: bool, + + /// Password used by the keystore. + #[structopt( + long = "password", + conflicts_with_all = &[ "password-interactive", "password-filename" ] + )] + pub password: Option, + + /// File that contains the password used by the keystore. + #[structopt( + long = "password-filename", + value_name = "PATH", + parse(from_os_str), + conflicts_with_all = &[ "password-interactive", "password" ] + )] + pub password_filename: Option, +} + +impl KeystoreParams { + /// Get the keystore configuration for the parameters + pub fn keystore_config(&self, base_path: &PathBuf) -> Result { + let password = if self.password_interactive { + #[cfg(not(target_os = "unknown"))] + { + Some(input_keystore_password()?.into()) + } + #[cfg(target_os = "unknown")] + None + } else if let Some(ref file) = self.password_filename { + Some( + fs::read_to_string(file) + .map_err(|e| format!("{}", e))? + .into(), + ) + } else if let Some(ref password) = self.password { + Some(password.clone().into()) + } else { + None + }; + + let path = self + .keystore_path + .clone() + .unwrap_or(base_path.join(DEFAULT_KEYSTORE_CONFIG_PATH)); + + Ok(KeystoreConfig::Path { path, password }) + } +} + +#[cfg(not(target_os = "unknown"))] +fn input_keystore_password() -> Result { + rpassword::read_password_from_tty(Some("Keystore password: ")) + .map_err(|e| format!("{:?}", e).into()) +} diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs index f684cab336423159c6a18ff492c3d7741c4cfded..9097bf8589919a25d2a118bab1e1011caa24dda1 100644 --- a/client/cli/src/params/mod.rs +++ b/client/cli/src/params/mod.rs @@ -15,21 +15,23 @@ // along with Substrate. If not, see . mod import_params; -mod transaction_pool_params; -mod shared_params; +mod keystore_params; +mod network_params; mod node_key_params; -mod network_configuration_params; mod pruning_params; +mod shared_params; +mod transaction_pool_params; -use std::str::FromStr; use std::fmt::Debug; +use std::str::FromStr; pub use crate::params::import_params::*; -pub use crate::params::transaction_pool_params::*; -pub use crate::params::shared_params::*; +pub use crate::params::keystore_params::*; +pub use crate::params::network_params::*; pub use crate::params::node_key_params::*; -pub use crate::params::network_configuration_params::*; pub use crate::params::pruning_params::*; +pub use crate::params::shared_params::*; +pub use crate::params::transaction_pool_params::*; /// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. #[derive(Debug, Clone)] diff --git a/client/cli/src/params/network_configuration_params.rs b/client/cli/src/params/network_params.rs similarity index 57% rename from client/cli/src/params/network_configuration_params.rs rename to client/cli/src/params/network_params.rs index 4de5e44fb5ace05f79b7c86739f2637b2fc650f0..2c008e19d1b0d30b6a2e0a84d08acc0088035872 100644 --- a/client/cli/src/params/network_configuration_params.rs +++ b/client/cli/src/params/network_params.rs @@ -14,28 +14,27 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::path::PathBuf; +use crate::params::node_key_params::NodeKeyParams; +use sc_network::{ + config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, TransportConfig}, + multiaddr::Protocol, +}; +use sc_service::{ChainSpec, config::{Multiaddr, MultiaddrWithPeerId}}; use std::iter; use std::net::Ipv4Addr; +use std::path::PathBuf; use structopt::StructOpt; -use sc_network::{ - config::{NonReservedPeerMode, TransportConfig}, multiaddr::Protocol, -}; -use sc_service::Configuration; - -use crate::error; -use crate::params::node_key_params::NodeKeyParams; /// Parameters used to create the network configuration. #[derive(Debug, StructOpt, Clone)] -pub struct NetworkConfigurationParams { +pub struct NetworkParams { /// Specify a list of bootnodes. - #[structopt(long = "bootnodes", value_name = "URL")] - pub bootnodes: Vec, + #[structopt(long = "bootnodes", value_name = "ADDR")] + pub bootnodes: Vec, /// Specify a list of reserved node addresses. - #[structopt(long = "reserved-nodes", value_name = "URL")] - pub reserved_nodes: Vec, + #[structopt(long = "reserved-nodes", value_name = "ADDR")] + pub reserved_nodes: Vec, /// Whether to only allow connections to/from reserved nodes. /// @@ -44,17 +43,9 @@ pub struct NetworkConfigurationParams { #[structopt(long = "reserved-only")] pub reserved_only: bool, - /// Specify a list of sentry node public addresses. - #[structopt( - long = "sentry-nodes", - value_name = "URL", - conflicts_with_all = &[ "sentry" ] - )] - pub sentry_nodes: Vec, - /// Listen on this multiaddress. #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] - pub listen_addr: Vec, + pub listen_addr: Vec, /// Specify p2p protocol TCP port. /// @@ -87,7 +78,11 @@ pub struct NetworkConfigurationParams { /// /// This allows downloading announced blocks from multiple peers. Decrease to save /// traffic and risk increased latency. - #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] + #[structopt( + long = "max-parallel-downloads", + value_name = "COUNT", + default_value = "5" + )] pub max_parallel_downloads: u32, #[allow(missing_docs)] @@ -99,59 +94,50 @@ pub struct NetworkConfigurationParams { pub use_yamux_flow_control: bool, } -impl NetworkConfigurationParams { +impl NetworkParams { /// Fill the given `NetworkConfiguration` by looking at the cli parameters. - pub fn update_config( + pub fn network_config( &self, - mut config: &mut Configuration, - config_path: PathBuf, - client_id: String, + chain_spec: &Box, is_dev: bool, - ) -> error::Result<()> { - config.network.boot_nodes.extend(self.bootnodes.clone()); - config.network.config_path = Some(config_path.clone()); - config.network.net_config_path = Some(config_path.clone()); - - config.network.reserved_nodes.extend(self.reserved_nodes.clone()); - if self.reserved_only { - config.network.non_reserved_mode = NonReservedPeerMode::Deny; - } - - config.network.sentry_nodes.extend(self.sentry_nodes.clone()); - - for addr in self.listen_addr.iter() { - let addr = addr.parse().ok().ok_or(error::Error::InvalidListenMultiaddress)?; - config.network.listen_addresses.push(addr); - } - - if config.network.listen_addresses.is_empty() { - let port = match self.port { - Some(port) => port, - None => 30333, - }; - - config.network.listen_addresses = vec![ - iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) - .chain(iter::once(Protocol::Tcp(port))) - .collect() - ]; + net_config_path: &PathBuf, + client_id: &str, + node_name: &str, + node_key: NodeKeyConfig, + ) -> NetworkConfiguration { + let port = self.port.unwrap_or(30333); + let mut listen_addresses = vec![iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) + .chain(iter::once(Protocol::Tcp(port))) + .collect()]; + + listen_addresses.extend(self.listen_addr.iter().cloned()); + + let mut boot_nodes = chain_spec.boot_nodes().to_vec(); + boot_nodes.extend(self.bootnodes.clone()); + + NetworkConfiguration { + boot_nodes, + net_config_path: net_config_path.clone(), + reserved_nodes: self.reserved_nodes.clone(), + non_reserved_mode: if self.reserved_only { + NonReservedPeerMode::Deny + } else { + NonReservedPeerMode::Accept + }, + listen_addresses, + public_addresses: Vec::new(), + node_key, + node_name: node_name.to_string(), + client_version: client_id.to_string(), + in_peers: self.in_peers, + out_peers: self.out_peers, + transport: TransportConfig::Normal { + enable_mdns: !is_dev && !self.no_mdns, + allow_private_ipv4: !self.no_private_ipv4, + wasm_external_transport: None, + use_yamux_flow_control: self.use_yamux_flow_control, + }, + max_parallel_downloads: self.max_parallel_downloads, } - - config.network.client_version = client_id; - self.node_key_params.update_config(&mut config, Some(&config_path))?; - - config.network.in_peers = self.in_peers; - config.network.out_peers = self.out_peers; - - config.network.transport = TransportConfig::Normal { - enable_mdns: !is_dev && !self.no_mdns, - allow_private_ipv4: !self.no_private_ipv4, - wasm_external_transport: None, - use_yamux_flow_control: self.use_yamux_flow_control, - }; - - config.network.max_parallel_downloads = self.max_parallel_downloads; - - Ok(()) } } diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs index c55ec8ee692db5715c88414c905f87fc20ca51fa..2913ff2c1035e2f04ae30888e9f352420c40331d 100644 --- a/client/cli/src/params/node_key_params.rs +++ b/client/cli/src/params/node_key_params.rs @@ -14,14 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{path::PathBuf, str::FromStr}; -use structopt::StructOpt; -use sc_service::Configuration; use sc_network::config::NodeKeyConfig; use sp_core::H256; +use std::{path::PathBuf, str::FromStr}; +use structopt::StructOpt; -use crate::error; use crate::arg_enums::NodeKeyType; +use crate::error; /// The file name of the node's Ed25519 secret key inside the chain-specific /// network config directory, if neither `--node-key` nor `--node-key-file` @@ -93,31 +92,23 @@ pub struct NodeKeyParams { impl NodeKeyParams { /// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context /// of an optional network config storage directory. - pub fn update_config<'a>( - &self, - mut config: &'a mut Configuration, - net_config_path: Option<&PathBuf>, - ) -> error::Result<&'a NodeKeyConfig> { - config.network.node_key = match self.node_key_type { + pub fn node_key(&self, net_config_dir: &PathBuf) -> error::Result { + Ok(match self.node_key_type { NodeKeyType::Ed25519 => { let secret = if let Some(node_key) = self.node_key.as_ref() { parse_ed25519_secret(node_key)? } else { - let path = self.node_key_file.clone() - .or_else(|| net_config_path.map(|d| d.join(NODE_KEY_ED25519_FILE))); + let path = self + .node_key_file + .clone() + .unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE)); - if let Some(path) = path { - sc_network::config::Secret::File(path) - } else { - sc_network::config::Secret::New - } + sc_network::config::Secret::File(path) }; NodeKeyConfig::Ed25519(secret) } - }; - - Ok(&config.network.node_key) + }) } } @@ -128,114 +119,107 @@ fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { /// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. fn parse_ed25519_secret(hex: &str) -> error::Result { - H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| - sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) - .map(sc_network::config::Secret::Input) - .map_err(invalid_node_key)) + H256::from_str(&hex) + .map_err(invalid_node_key) + .and_then(|bytes| { + sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) + .map(sc_network::config::Secret::Input) + .map_err(invalid_node_key) + }) } #[cfg(test)] mod tests { - use sc_network::config::identity::ed25519; use super::*; + use sc_network::config::identity::ed25519; #[test] fn test_node_key_config_input() { - fn secret_input(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + fn secret_input(net_config_dir: &PathBuf) -> error::Result<()> { NodeKeyType::variants().iter().try_for_each(|t| { - let mut config = Configuration::default(); let node_key_type = NodeKeyType::from_str(t).unwrap(); let sk = match node_key_type { - NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() + NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec(), }; let params = NodeKeyParams { node_key_type, node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), - node_key_file: None + node_key_file: None, }; - params.update_config(&mut config, net_config_dir).and_then(|c| match c { + params.node_key(net_config_dir).and_then(|c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) - if node_key_type == NodeKeyType::Ed25519 && - &sk[..] == ski.as_ref() => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) + if node_key_type == NodeKeyType::Ed25519 && &sk[..] == ski.as_ref() => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), }) }) } - assert!(secret_input(None).is_ok()); - assert!(secret_input(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + assert!(secret_input(&PathBuf::from_str("x").unwrap()).is_ok()); } #[test] fn test_node_key_config_file() { - fn secret_file(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + fn secret_file(net_config_dir: &PathBuf) -> error::Result<()> { NodeKeyType::variants().iter().try_for_each(|t| { - let mut config = Configuration::default(); let node_key_type = NodeKeyType::from_str(t).unwrap(); let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); let params = NodeKeyParams { node_key_type, node_key: None, - node_key_file: Some(file.clone()) + node_key_file: Some(file.clone()), }; - params.update_config(&mut config, net_config_dir).and_then(|c| match c { + params.node_key(net_config_dir).and_then(|c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) + if node_key_type == NodeKeyType::Ed25519 && f == &file => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), }) }) } - assert!(secret_file(None).is_ok()); - assert!(secret_file(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + assert!(secret_file(&PathBuf::from_str("x").unwrap()).is_ok()); } #[test] fn test_node_key_config_default() { fn with_def_params(f: F) -> error::Result<()> where - F: Fn(NodeKeyParams) -> error::Result<()> + F: Fn(NodeKeyParams) -> error::Result<()>, { NodeKeyType::variants().iter().try_for_each(|t| { let node_key_type = NodeKeyType::from_str(t).unwrap(); f(NodeKeyParams { node_key_type, node_key: None, - node_key_file: None + node_key_file: None, }) }) } - fn no_config_dir() -> error::Result<()> { - with_def_params(|params| { - let mut config = Configuration::default(); - let typ = params.node_key_type; - params.update_config(&mut config, None) - .and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::New) - if typ == NodeKeyType::Ed25519 => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - fn some_config_dir(net_config_dir: &PathBuf) -> error::Result<()> { with_def_params(|params| { - let mut config = Configuration::default(); let dir = PathBuf::from(net_config_dir.clone()); let typ = params.node_key_type; - params.update_config(&mut config, Some(net_config_dir)) + params + .node_key(net_config_dir) .and_then(move |c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if typ == NodeKeyType::Ed25519 && - f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) + if typ == NodeKeyType::Ed25519 + && f == &dir.join(NODE_KEY_ED25519_FILE) => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), + }) }) } - assert!(no_config_dir().is_ok()); assert!(some_config_dir(&PathBuf::from_str("x").unwrap()).is_ok()); } } diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index ec1066df953fdc01538e089af632aeaa128358d5..ed8f7ab16858dce49f0b8a78ae2bb60141a47eac 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -14,10 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use sc_service::{Configuration, PruningMode}; - use crate::error; +use sc_service::{PruningMode, Role}; +use structopt::StructOpt; /// Parameters to define the pruning mode #[derive(Debug, StructOpt, Clone)] @@ -32,35 +31,29 @@ pub struct PruningParams { } impl PruningParams { - /// Put block pruning CLI params into `config` object. - pub fn update_config( - &self, - mut config: &mut Configuration, - role: sc_service::Roles, - unsafe_pruning: bool, - ) -> error::Result<()> { + /// Get the pruning value from the parameters + pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result { // by default we disable pruning if the node is an authority (i.e. // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the // node is an authority and pruning is enabled explicitly, then we error // unless `unsafe_pruning` is set. - config.pruning = match &self.pruning { + Ok(match &self.pruning { Some(ref s) if s == "archive" => PruningMode::ArchiveAll, - None if role == sc_service::Roles::AUTHORITY => PruningMode::ArchiveAll, + None if role.is_network_authority() => PruningMode::ArchiveAll, None => PruningMode::default(), Some(s) => { - if role == sc_service::Roles::AUTHORITY && !unsafe_pruning { + if role.is_network_authority() && !unsafe_pruning { return Err(error::Error::Input( "Validators should run with state pruning disabled (i.e. archive). \ - You can ignore this check with `--unsafe-pruning`.".to_string() + You can ignore this check with `--unsafe-pruning`." + .to_string(), )); } - PruningMode::keep_blocks(s.parse() - .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? - ) - }, - }; - - Ok(()) + PruningMode::keep_blocks(s.parse().map_err(|_| { + error::Error::Input("Invalid pruning mode specified".to_string()) + })?) + } + }) } } diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs index 41b9cce8264cc0ce46de7d81661b396f4275266b..f7f9db102c714dfb5e33a8ed54468f7dccee0fe3 100644 --- a/client/cli/src/params/shared_params.rs +++ b/client/cli/src/params/shared_params.rs @@ -14,18 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use sc_service::config::DatabaseConfig; use std::path::PathBuf; use structopt::StructOpt; -use app_dirs::{AppInfo, AppDataType}; -use sc_service::{ - Configuration, config::DatabaseConfig, ChainSpec, -}; - -use crate::VersionInfo; -use crate::error; /// default sub directory to store database -const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; +const DEFAULT_DB_CONFIG_PATH: &'static str = "db"; /// Shared parameters used by all `CoreParams`. #[derive(Debug, StructOpt, Clone)] @@ -39,7 +33,12 @@ pub struct SharedParams { pub dev: bool, /// Specify custom base path. - #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] + #[structopt( + long = "base-path", + short = "d", + value_name = "PATH", + parse(from_os_str) + )] pub base_path: Option, /// Sets a custom logging filter. Syntax is =, e.g. -lsync=debug. @@ -51,62 +50,44 @@ pub struct SharedParams { } impl SharedParams { - /// Load spec to `Configuration` from `SharedParams` and spec factory. - pub fn update_config<'a, F>( - &self, - mut config: &'a mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<&'a dyn ChainSpec> where - F: FnOnce(&str) -> Result, String>, - { - let chain_key = match self.chain { - Some(ref chain) => chain.clone(), - None => if self.dev { "dev".into() } else { "".into() } - }; - let spec = spec_factory(&chain_key)?; - config.network.boot_nodes = spec.boot_nodes().to_vec(); - config.telemetry_endpoints = spec.telemetry_endpoints().clone(); + /// Specify custom base path. + pub fn base_path(&self) -> Option { + self.base_path.clone() + } - config.chain_spec = Some(spec); + /// Specify the development chain. + pub fn is_dev(&self) -> bool { + self.dev + } - if config.config_dir.is_none() { - config.config_dir = Some(base_path(self, version)); + /// Get the chain spec for the parameters provided + pub fn chain_id(&self, is_dev: bool) -> String { + match self.chain { + Some(ref chain) => chain.clone(), + None => { + if is_dev { + "dev".into() + } else { + "".into() + } + } } + } - if config.database.is_none() { - config.database = Some(DatabaseConfig::Path { - path: config - .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) - .expect("We provided a base_path/config_dir."), - cache_size: None, - }); + /// Get the database configuration object for the parameters provided + pub fn database_config( + &self, + base_path: &PathBuf, + cache_size: usize, + ) -> DatabaseConfig { + DatabaseConfig::Path { + path: base_path.join(DEFAULT_DB_CONFIG_PATH), + cache_size, } - - Ok(config.expect_chain_spec()) } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - crate::init(self.log.as_ref().map(|v| v.as_ref()).unwrap_or(""), version) + /// Get the filters for the logging + pub fn log_filters(&self) -> Option { + self.log.clone() } } - -fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { - cli.base_path.clone() - .unwrap_or_else(|| - app_dirs::get_app_root( - AppDataType::UserData, - &AppInfo { - name: version.executable_name, - author: version.author - } - ).expect("app directories exist on all supported platforms; qed") - ) -} diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs index 3468c12243b3fe75d872891aebf12b3df6f0e356..dfcdf9af70537e51586654e4cc72da4a7549f118 100644 --- a/client/cli/src/params/transaction_pool_params.rs +++ b/client/cli/src/params/transaction_pool_params.rs @@ -14,9 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use sc_service::config::TransactionPoolOptions; use structopt::StructOpt; -use sc_service::Configuration; -use crate::error; /// Parameters used to create the pool configuration. #[derive(Debug, StructOpt, Clone)] @@ -24,6 +23,7 @@ pub struct TransactionPoolParams { /// Maximum number of transactions in the transaction pool. #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] pub pool_limit: usize, + /// Maximum number of kilobytes of all transactions stored in the pool. #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] pub pool_kbytes: usize, @@ -31,19 +31,18 @@ pub struct TransactionPoolParams { impl TransactionPoolParams { /// Fill the given `PoolConfiguration` by looking at the cli parameters. - pub fn update_config( - &self, - config: &mut Configuration, - ) -> error::Result<()> { + pub fn transaction_pool(&self) -> TransactionPoolOptions { + let mut opts = TransactionPoolOptions::default(); + // ready queue - config.transaction_pool.ready.count = self.pool_limit; - config.transaction_pool.ready.total_bytes = self.pool_kbytes * 1024; + opts.ready.count = self.pool_limit; + opts.ready.total_bytes = self.pool_kbytes * 1024; // future queue let factor = 10; - config.transaction_pool.future.count = self.pool_limit / factor; - config.transaction_pool.future.total_bytes = self.pool_kbytes * 1024 / factor; + opts.future.count = self.pool_limit / factor; + opts.future.total_bytes = self.pool_kbytes * 1024 / factor; - Ok(()) + opts } } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs new file mode 100644 index 0000000000000000000000000000000000000000..bd5dc7100ef02a58855032d876108a0a1573e6c0 --- /dev/null +++ b/client/cli/src/runner.rs @@ -0,0 +1,237 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::CliConfiguration; +use crate::Result; +use crate::SubstrateCli; +use crate::Subcommand; +use chrono::prelude::*; +use futures::pin_mut; +use futures::select; +use futures::{future, future::FutureExt, Future}; +use log::info; +use sc_service::{AbstractService, Configuration, Role, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::Arc; + +#[cfg(target_family = "unix")] +async fn main(func: F) -> std::result::Result<(), Box> +where + F: Future> + future::FusedFuture, + E: 'static + std::error::Error, +{ + use tokio::signal::unix::{signal, SignalKind}; + + let mut stream_int = signal(SignalKind::interrupt())?; + let mut stream_term = signal(SignalKind::terminate())?; + + let t1 = stream_int.recv().fuse(); + let t2 = stream_term.recv().fuse(); + let t3 = func; + + pin_mut!(t1, t2, t3); + + select! { + _ = t1 => {}, + _ = t2 => {}, + res = t3 => res?, + } + + Ok(()) +} + +#[cfg(not(unix))] +async fn main(func: F) -> std::result::Result<(), Box> +where + F: Future> + future::FusedFuture, + E: 'static + std::error::Error, +{ + use tokio::signal::ctrl_c; + + let t1 = ctrl_c().fuse(); + let t2 = func; + + pin_mut!(t1, t2); + + select! { + _ = t1 => {}, + res = t2 => res?, + } + + Ok(()) +} + +/// Build a tokio runtime with all features +pub fn build_runtime() -> std::result::Result { + tokio::runtime::Builder::new() + .thread_name("main-tokio-") + .threaded_scheduler() + .on_thread_start(||{ + TOKIO_THREADS_ALIVE.inc(); + TOKIO_THREADS_TOTAL.inc(); + }) + .on_thread_stop(||{ + TOKIO_THREADS_ALIVE.dec(); + }) + .enable_all() + .build() +} + +fn run_until_exit(mut tokio_runtime: tokio::runtime::Runtime, future: FUT) -> Result<()> +where + FUT: Future> + future::Future, + ERR: 'static + std::error::Error, +{ + let f = future.fuse(); + pin_mut!(f); + + tokio_runtime.block_on(main(f)).map_err(|e| e.to_string())?; + + Ok(()) +} + +/// A Substrate CLI runtime that can be used to run a node or a command +pub struct Runner { + config: Configuration, + tokio_runtime: tokio::runtime::Runtime, + phantom: PhantomData, +} + +impl Runner { + /// Create a new runtime with the command provided in argument + pub fn new(cli: &C, command: &T) -> Result> { + let tokio_runtime = build_runtime()?; + + let task_executor = { + let runtime_handle = tokio_runtime.handle().clone(); + Arc::new(move |fut| { + runtime_handle.spawn(fut); + }) + }; + + Ok(Runner { + config: command.create_configuration(cli, task_executor)?, + tokio_runtime, + phantom: PhantomData, + }) + } + + /// A helper function that runs an `AbstractService` with tokio and stops if the process receives + /// the signal `SIGTERM` or `SIGINT`. + pub fn run_node(self, new_light: FNL, new_full: FNF) -> Result<()> + where + FNL: FnOnce(Configuration) -> sc_service::error::Result, + FNF: FnOnce(Configuration) -> sc_service::error::Result, + SL: AbstractService + Unpin, + SF: AbstractService + Unpin, + { + info!("{}", C::impl_name()); + info!("✌️ version {}", C::impl_version()); + info!( + "❤️ by {}, {}-{}", + C::author(), + C::copyright_start_year(), + Local::today().year(), + ); + info!("📋 Chain specification: {}", self.config.chain_spec.name()); + info!("🏷 Node name: {}", self.config.network.node_name); + info!("👤 Role: {}", self.config.display_role()); + + match self.config.role { + Role::Light => self.run_service_until_exit(new_light), + _ => self.run_service_until_exit(new_full), + } + } + + /// A helper function that runs a future with tokio and stops if the process receives the signal + /// `SIGTERM` or `SIGINT`. + pub fn run_subcommand(self, subcommand: &Subcommand, builder: B) -> Result<()> + where + B: FnOnce(Configuration) -> sc_service::error::Result, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: Debug, + ::Hash: std::str::FromStr, + { + match subcommand { + Subcommand::BuildSpec(cmd) => cmd.run(self.config), + Subcommand::ExportBlocks(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::ImportBlocks(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::CheckBlock(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::Revert(cmd) => cmd.run(self.config, builder), + Subcommand::PurgeChain(cmd) => cmd.run(self.config), + } + } + + fn run_service_until_exit(mut self, service_builder: F) -> Result<()> + where + F: FnOnce(Configuration) -> std::result::Result, + T: AbstractService + Unpin, + { + let service = service_builder(self.config)?; + + let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); + let _informant_handle = self.tokio_runtime.spawn(informant_future); + + // we eagerly drop the service so that the internal exit future is fired, + // but we need to keep holding a reference to the global telemetry guard + // and drop the runtime first. + let _telemetry = service.telemetry(); + + let f = service.fuse(); + pin_mut!(f); + + self.tokio_runtime + .block_on(main(f)) + .map_err(|e| e.to_string())?; + drop(self.tokio_runtime); + + Ok(()) + } + + /// A helper function that runs a command with the configuration of this node + pub fn sync_run(self, runner: impl FnOnce(Configuration) -> Result<()>) -> Result<()> { + runner(self.config) + } + + /// A helper function that runs a future with tokio and stops if the process receives + /// the signal SIGTERM or SIGINT + pub fn async_run(self, runner: impl FnOnce(Configuration) -> FUT) -> Result<()> + where + FUT: Future>, + { + run_until_exit(self.tokio_runtime, runner(self.config)) + } + + /// Get an immutable reference to the node Configuration + pub fn config(&self) -> &Configuration { + &self.config + } + + /// Get a mutable reference to the node Configuration + pub fn config_mut(&mut self) -> &Configuration { + &mut self.config + } +} diff --git a/client/cli/src/runtime.rs b/client/cli/src/runtime.rs deleted file mode 100644 index 183196139ffb05953b010e4a7028cced82cb3dba..0000000000000000000000000000000000000000 --- a/client/cli/src/runtime.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use std::sync::Arc; - -use futures::{Future, future, future::FutureExt}; -use futures::select; -use futures::pin_mut; -use sc_service::{AbstractService, Configuration}; -use crate::error; - -#[cfg(target_family = "unix")] -async fn main(func: F) -> Result<(), Box> -where - F: Future> + future::FusedFuture, - E: 'static + std::error::Error, -{ - use tokio::signal::unix::{signal, SignalKind}; - - let mut stream_int = signal(SignalKind::interrupt())?; - let mut stream_term = signal(SignalKind::terminate())?; - - let t1 = stream_int.recv().fuse(); - let t2 = stream_term.recv().fuse(); - let t3 = func; - - pin_mut!(t1, t2, t3); - - select! { - _ = t1 => {}, - _ = t2 => {}, - res = t3 => res?, - } - - Ok(()) -} - -#[cfg(not(unix))] -async fn main(func: F) -> Result<(), Box> -where - F: Future> + future::FusedFuture, - E: 'static + std::error::Error, -{ - use tokio::signal::ctrl_c; - - let t1 = ctrl_c().fuse(); - let t2 = func; - - pin_mut!(t1, t2); - - select! { - _ = t1 => {}, - res = t2 => res?, - } - - Ok(()) -} - -fn build_runtime() -> Result { - tokio::runtime::Builder::new() - .thread_name("main-tokio-") - .threaded_scheduler() - .enable_all() - .build() -} - -/// A helper function that runs a future with tokio and stops if the process receives the signal -/// SIGTERM or SIGINT -pub fn run_until_exit( - mut config: Configuration, - future_builder: F, -) -> error::Result<()> -where - F: FnOnce(Configuration) -> error::Result, - FUT: Future> + future::Future, - ERR: 'static + std::error::Error, -{ - let mut runtime = build_runtime()?; - - config.task_executor = { - let runtime_handle = runtime.handle().clone(); - Some(Arc::new(move |fut| { runtime_handle.spawn(fut); })) - }; - - let f = future_builder(config)?; - let f = f.fuse(); - pin_mut!(f); - - runtime.block_on(main(f)).map_err(|e| e.to_string())?; - - Ok(()) -} - -/// A helper function that runs an `AbstractService` with tokio and stops if the process receives -/// the signal SIGTERM or SIGINT -pub fn run_service_until_exit( - mut config: Configuration, - service_builder: F, -) -> error::Result<()> -where - F: FnOnce(Configuration) -> Result, - T: AbstractService + Unpin, -{ - let mut runtime = build_runtime()?; - - config.task_executor = { - let runtime_handle = runtime.handle().clone(); - Some(Arc::new(move |fut| { runtime_handle.spawn(fut); })) - }; - - let service = service_builder(config)?; - - let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); - let _informant_handle = runtime.spawn(informant_future); - - // we eagerly drop the service so that the internal exit future is fired, - // but we need to keep holding a reference to the global telemetry guard - // and drop the runtime first. - let _telemetry = service.telemetry(); - - let f = service.fuse(); - pin_mut!(f); - - runtime.block_on(main(f)).map_err(|e| e.to_string())?; - drop(runtime); - - Ok(()) -} diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index ced86173756980a389c471d49a25994843648737..f0fe368acbb037f0b9181263889baa31d3e3f660 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -15,10 +15,10 @@ sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/bloc sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } sc-client = { version = "0.8.0-alpha.5", path = "../../" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } sc-keystore = { version = "2.0.0-alpha.5", path = "../../keystore" } @@ -43,3 +43,6 @@ sc-service = { version = "0.8.0-alpha.5", path = "../../service" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } env_logger = "0.7.0" tempfile = "3.1.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index a0d73180ed177aad616c2ee3f0a4431168f24988..19c5bf9e588b3d51f933855ddd01e1a80cd6eae9 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-consensus-babe" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/babe" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../primitives/application-crypto" } @@ -36,7 +36,7 @@ sc-consensus-uncles = { version = "0.8.0-alpha.5", path = "../uncles" } sc-consensus-slots = { version = "0.8.0-alpha.5", path = "../slots" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } fork-tree = { version = "2.0.0-alpha.5", path = "../../../utils/fork-tree" } -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" @@ -59,3 +59,6 @@ tempfile = "3.1.0" [features] test-helpers = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 1f820cd66bd01898f79924ce1cfe70188d52a8e3..6bdaeb5ddf0709a336b0bccf6d30698ba810d2dd 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -11,14 +11,14 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] sc-consensus-babe = { version = "0.8.0-alpha.5", path = "../" } jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" +jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../../primitives/consensus/babe" } serde = { version = "1.0.104", features=["derive"] } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../../primitives/blockchain" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../../primitives/runtime" } sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../epochs" } -futures = "0.3.1" +futures = "0.3.4" derive_more = "0.99.2" sp-api = { version = "2.0.0-alpha.5", path = "../../../../primitives/api" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../../primitives/consensus/common" } @@ -30,3 +30,6 @@ substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../tes sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../../primitives/application-crypto" } sp-keyring = { version = "2.0.0-alpha.5", path = "../../../../primitives/keyring" } tempfile = "3.1.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index fb7be52766c70d3b93a03ac8842a8f1f82a9b1f9..074e582bff252395a18e691211eda944b380c723 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -21,7 +21,7 @@ use sp_consensus_babe::{ AuthorityId, BabeAuthorityWeight, BABE_ENGINE_ID, BABE_VRF_PREFIX, SlotNumber, AuthorityPair, BabeConfiguration }; -use sp_consensus_babe::digests::PreDigest; +use sp_consensus_babe::digests::{PreDigest, PrimaryPreDigest, SecondaryPreDigest}; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; use sp_core::{U256, blake2_256}; use codec::Encode; @@ -128,10 +128,10 @@ fn claim_secondary_slot( }) { if pair.public() == *expected_author { - let pre_digest = PreDigest::Secondary { + let pre_digest = PreDigest::Secondary(SecondaryPreDigest { slot_number, authority_index: authority_index as u32, - }; + }); return Some((pre_digest, pair)); } @@ -200,12 +200,12 @@ fn claim_primary_slot( let pre_digest = get_keypair(&pair) .vrf_sign_after_check(transcript, |inout| super::authorship::check_primary_threshold(inout, threshold)) .map(|s| { - PreDigest::Primary { + PreDigest::Primary(PrimaryPreDigest { slot_number, vrf_output: VRFOutput(s.0.to_output()), vrf_proof: VRFProof(s.1), authority_index: authority_index as u32, - } + }) }); // early exit on first successful claim diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs index 3ca2f560d43c7f1d309e70df47b40cc2e6f156da..e014c8975acee02b1efddae059345b08cd395d6c 100644 --- a/client/consensus/babe/src/aux_schema.rs +++ b/client/consensus/babe/src/aux_schema.rs @@ -25,7 +25,7 @@ use sc_client_api::backend::AuxStore; use sp_blockchain::{Result as ClientResult, Error as ClientError}; use sp_runtime::traits::Block as BlockT; use sp_consensus_babe::BabeBlockWeight; -use sc_consensus_epochs::{EpochChangesFor, SharedEpochChanges}; +use sc_consensus_epochs::{EpochChangesFor, SharedEpochChanges, migration::EpochChangesForV0}; use crate::Epoch; const BABE_EPOCH_CHANGES_VERSION: &[u8] = b"babe_epoch_changes_version"; @@ -57,7 +57,11 @@ pub(crate) fn load_epoch_changes( let version = load_decode::<_, u32>(backend, BABE_EPOCH_CHANGES_VERSION)?; let maybe_epoch_changes = match version { - None | Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) => load_decode::<_, EpochChangesFor>( + None => load_decode::<_, EpochChangesForV0>( + backend, + BABE_EPOCH_CHANGES_KEY, + )?.map(|v0| v0.migrate()), + Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) => load_decode::<_, EpochChangesFor>( backend, BABE_EPOCH_CHANGES_KEY, )?, @@ -70,7 +74,7 @@ pub(crate) fn load_epoch_changes( let epoch_changes = Arc::new(Mutex::new(maybe_epoch_changes.unwrap_or_else(|| { info!(target: "babe", - "Creating empty BABE epoch changes on what appears to be first startup." + "👶 Creating empty BABE epoch changes on what appears to be first startup." ); EpochChangesFor::::default() }))); @@ -123,3 +127,72 @@ pub(crate) fn load_block_weight( ) -> ClientResult> { load_decode(backend, block_weight_key(block_hash).as_slice()) } + +#[cfg(test)] +mod test { + use super::*; + use crate::Epoch; + use fork_tree::ForkTree; + use substrate_test_runtime_client; + use sp_core::H256; + use sp_runtime::traits::NumberFor; + use sc_consensus_epochs::{PersistedEpoch, PersistedEpochHeader, EpochHeader}; + use sp_consensus::Error as ConsensusError; + use sc_network_test::Block as TestBlock; + + #[test] + fn load_decode_from_v0_epoch_changes() { + let epoch = Epoch { + start_slot: 0, + authorities: vec![], + randomness: [0; 32], + epoch_index: 1, + duration: 100, + }; + let client = substrate_test_runtime_client::new(); + let mut v0_tree = ForkTree::, _>::new(); + v0_tree.import::<_, ConsensusError>( + Default::default(), + Default::default(), + PersistedEpoch::Regular(epoch), + &|_, _| Ok(false), // Test is single item only so this can be set to false. + ).unwrap(); + + client.insert_aux( + &[(BABE_EPOCH_CHANGES_KEY, + &EpochChangesForV0::::from_raw(v0_tree).encode()[..])], + &[], + ).unwrap(); + + assert_eq!( + load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), + None, + ); + + let epoch_changes = load_epoch_changes::(&client).unwrap(); + + assert!( + epoch_changes.lock() + .tree() + .iter() + .map(|(_, _, epoch)| epoch.clone()) + .collect::>() == + vec![PersistedEpochHeader::Regular(EpochHeader { + start_slot: 0, + end_slot: 100, + })], + ); // PersistedEpochHeader does not implement Debug, so we use assert! directly. + + write_epoch_changes::( + &epoch_changes.lock(), + |values| { + client.insert_aux(values, &[]).unwrap(); + }, + ); + + assert_eq!( + load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), + Some(1), + ); + } +} diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index f4373da47f26cca9145e93316a1c6692555cf719..092bf8153b9bff292bc4aa5cbd86505907219a59 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -62,7 +62,9 @@ pub use sp_consensus_babe::{ BabeApi, ConsensusLog, BABE_ENGINE_ID, SlotNumber, BabeConfiguration, AuthorityId, AuthorityPair, AuthoritySignature, BabeAuthorityWeight, VRF_OUTPUT_LENGTH, - digests::{PreDigest, CompatibleDigestItem, NextEpochDescriptor}, + digests::{ + CompatibleDigestItem, NextEpochDescriptor, PreDigest, PrimaryPreDigest, SecondaryPreDigest, + }, }; pub use sp_consensus::SyncOracle; use std::{ @@ -99,7 +101,7 @@ use sc_client_api::{ use sp_block_builder::BlockBuilder as BlockBuilderApi; use futures::prelude::*; -use log::{warn, debug, info, trace}; +use log::{debug, info, log, trace, warn}; use sc_consensus_slots::{ SlotWorker, SlotInfo, SlotCompatible, StorageChanges, CheckedHeader, check_equivocation, }; @@ -220,16 +222,6 @@ fn babe_err(error: Error) -> Error { error } -macro_rules! babe_info { - ($($i: expr),+) => { - { - info!(target: "babe", $($i),+); - format!($($i),+) - } - }; -} - - /// Intermediate value passed to block importer. pub struct BabeIntermediate { /// The epoch descriptor. @@ -366,7 +358,7 @@ pub fn start_babe(BabeParams { &inherent_data_providers, )?; - babe_info!("Starting BABE Authorship worker"); + info!(target: "babe", "👶 Starting BABE Authorship worker"); Ok(sc_consensus_slots::start_slot_worker( config.0, select_chain, @@ -580,10 +572,10 @@ fn find_pre_digest(header: &B::Header) -> Result> // genesis block doesn't contain a pre digest so let's generate a // dummy one to not break any invariants in the rest of the code if header.number().is_zero() { - return Ok(PreDigest::Secondary { + return Ok(PreDigest::Secondary(SecondaryPreDigest { slot_number: 0, authority_index: 0, - }); + })); } let mut pre_digest: Option<_> = None; @@ -1008,15 +1000,29 @@ impl BlockImport for BabeBlockImport::FetchEpoch(parent_hash).into()) })?; - babe_info!("New epoch {} launching at block {} (block slot {} >= start slot {}).", - viable_epoch.as_ref().epoch_index, - hash, - slot_number, - viable_epoch.as_ref().start_slot); + // restrict info logging during initial sync to avoid spam + let log_level = if block.origin == BlockOrigin::NetworkInitialSync { + log::Level::Debug + } else { + log::Level::Info + }; + + log!(target: "babe", + log_level, + "👶 New epoch {} launching at block {} (block slot {} >= start slot {}).", + viable_epoch.as_ref().epoch_index, + hash, + slot_number, + viable_epoch.as_ref().start_slot, + ); let next_epoch = viable_epoch.increment(next_epoch_descriptor); - babe_info!("Next epoch starts at slot {}", next_epoch.as_ref().start_slot); + log!(target: "babe", + log_level, + "👶 Next epoch starts at slot {}", + next_epoch.as_ref().start_slot, + ); // prune the tree of epochs not part of the finalized chain or // that are not live anymore, and then track the given epoch change diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index ec5a6032f5f3489876c573353046d68dc0e392a2..20b924669d6147816c0fbf9c0c8e85937b17c1af 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -553,10 +553,10 @@ fn propose_and_import_block( let pre_digest = sp_runtime::generic::Digest { logs: vec![ Item::babe_pre_digest( - PreDigest::Secondary { + PreDigest::Secondary(SecondaryPreDigest { authority_index: 0, slot_number, - }, + }), ), ], }; diff --git a/client/consensus/babe/src/verification.rs b/client/consensus/babe/src/verification.rs index 1eb73588a87bcad63cafe09e2f4ed8972bbe094d..2fd37280b3b369bdb1e73895dc9ae683030eaa23 100644 --- a/client/consensus/babe/src/verification.rs +++ b/client/consensus/babe/src/verification.rs @@ -17,9 +17,10 @@ //! Verification for BABE headers. use sp_runtime::{traits::Header, traits::DigestItemFor}; use sp_core::{Pair, Public}; -use sp_consensus_babe::{AuthoritySignature, SlotNumber, AuthorityIndex, AuthorityPair, AuthorityId}; -use sp_consensus_babe::digests::{PreDigest, CompatibleDigestItem}; -use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; +use sp_consensus_babe::{AuthoritySignature, SlotNumber, AuthorityPair, AuthorityId}; +use sp_consensus_babe::digests::{ + PreDigest, PrimaryPreDigest, SecondaryPreDigest, CompatibleDigestItem +}; use sc_consensus_slots::CheckedHeader; use log::{debug, trace}; use super::{find_pre_digest, babe_err, Epoch, BlockT, Error}; @@ -93,27 +94,23 @@ pub(super) fn check_header( }; match &pre_digest { - PreDigest::Primary { vrf_output, vrf_proof, authority_index, slot_number } => { + PreDigest::Primary(primary) => { debug!(target: "babe", "Verifying Primary block"); - let digest = (vrf_output, vrf_proof, *authority_index, *slot_number); - check_primary_header::( pre_hash, - digest, + primary, sig, &epoch, config.c, )?; }, - PreDigest::Secondary { authority_index, slot_number } if config.secondary_slots => { + PreDigest::Secondary(secondary) if config.secondary_slots => { debug!(target: "babe", "Verifying Secondary block"); - let digest = (*authority_index, *slot_number); - check_secondary_header::( pre_hash, - digest, + secondary, sig, &epoch, )?; @@ -143,25 +140,23 @@ pub(super) struct VerifiedHeaderInfo { /// its parent since it is a primary block. fn check_primary_header( pre_hash: B::Hash, - pre_digest: (&VRFOutput, &VRFProof, AuthorityIndex, SlotNumber), + pre_digest: &PrimaryPreDigest, signature: AuthoritySignature, epoch: &Epoch, c: (u64, u64), ) -> Result<(), Error> { - let (vrf_output, vrf_proof, authority_index, slot_number) = pre_digest; - - let author = &epoch.authorities[authority_index as usize].0; + let author = &epoch.authorities[pre_digest.authority_index as usize].0; if AuthorityPair::verify(&signature, pre_hash, &author) { let (inout, _) = { let transcript = make_transcript( &epoch.randomness, - slot_number, + pre_digest.slot_number, epoch.epoch_index, ); schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| { - p.vrf_verify(transcript, vrf_output, vrf_proof) + p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof) }).map_err(|s| { babe_err(Error::VRFVerificationFailed(s)) })? @@ -170,7 +165,7 @@ fn check_primary_header( let threshold = calculate_primary_threshold( c, &epoch.authorities, - authority_index as usize, + pre_digest.authority_index as usize, ); if !check_primary_threshold(&inout, threshold) { @@ -189,21 +184,19 @@ fn check_primary_header( /// compared to its parent since it is a secondary block. fn check_secondary_header( pre_hash: B::Hash, - pre_digest: (AuthorityIndex, SlotNumber), + pre_digest: &SecondaryPreDigest, signature: AuthoritySignature, epoch: &Epoch, ) -> Result<(), Error> { - let (authority_index, slot_number) = pre_digest; - // check the signature is valid under the expected authority and // chain state. let expected_author = secondary_slot_author( - slot_number, + pre_digest.slot_number, &epoch.authorities, epoch.randomness, ).ok_or_else(|| Error::NoSecondaryAuthorExpected)?; - let author = &epoch.authorities[authority_index as usize].0; + let author = &epoch.authorities[pre_digest.authority_index as usize].0; if expected_author != author { return Err(Error::InvalidAuthor(expected_author.clone(), author.clone())); diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index acc13bd57589fbd999e5b78de186a3516d7a53c0..3496141ec712bdc4c8daa043e10faaecb13d8b7e 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -9,9 +9,12 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" fork-tree = { version = "2.0.0-alpha.5", path = "../../../utils/fork-tree" } sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.5"} sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } sc-client-api = { path = "../../api" , version = "2.0.0-alpha.5"} + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/epochs/src/lib.rs b/client/consensus/epochs/src/lib.rs index d5816d960cc9eaaa0dabd16ee11899ac220d84d5..001c172b3490573dea6a0dcfbefc428cf79c13aa 100644 --- a/client/consensus/epochs/src/lib.rs +++ b/client/consensus/epochs/src/lib.rs @@ -16,6 +16,8 @@ //! Generic utilities for epoch-based consensus engines. +pub mod migration; + use std::{sync::Arc, ops::Add, collections::BTreeMap, borrow::{Borrow, BorrowMut}}; use parking_lot::Mutex; use codec::{Encode, Decode}; diff --git a/client/consensus/epochs/src/migration.rs b/client/consensus/epochs/src/migration.rs new file mode 100644 index 0000000000000000000000000000000000000000..e4717b5584e0ec67801ce3d264e90ff350286deb --- /dev/null +++ b/client/consensus/epochs/src/migration.rs @@ -0,0 +1,55 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Migration types for epoch changes. + +use std::collections::BTreeMap; +use codec::{Encode, Decode}; +use fork_tree::ForkTree; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use crate::{Epoch, EpochChanges, PersistedEpoch, PersistedEpochHeader}; + +/// Legacy definition of epoch changes. +#[derive(Clone, Encode, Decode)] +pub struct EpochChangesV0 { + inner: ForkTree>, +} + +/// Type alias for legacy definition of epoch changes. +pub type EpochChangesForV0 = EpochChangesV0<::Hash, NumberFor, Epoch>; + +impl EpochChangesV0 where + Hash: PartialEq + Ord + Copy, + Number: Ord + Copy, +{ + /// Create a new value of this type from raw. + pub fn from_raw(inner: ForkTree>) -> Self { + Self { inner } + } + + /// Migrate the type into current epoch changes definition. + pub fn migrate(self) -> EpochChanges { + let mut epochs = BTreeMap::new(); + + let inner = self.inner.map(&mut |hash, number, data| { + let header = PersistedEpochHeader::from(&data); + epochs.insert((*hash, *number), data); + header + }); + + EpochChanges { inner, epochs } + } +} diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index edb98ab21aaeb6529d61702d6b8fe9c589bccf3b..b7e5f7b0b1a9ff7c3f14303fa8c0065aa572048d 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" -futures = "0.3.1" +futures = "0.3.4" jsonrpc-core = "14.0.5" jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.5" @@ -35,3 +35,6 @@ substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/ tokio = { version = "0.2", features = ["rt-core", "macros"] } env_logger = "0.7.0" tempfile = "3.1.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index 419159709f2db84ae51cba0fb861458123b63332..5780a25f97256331eb1d11b2370dd838ec112ac7 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -23,42 +23,47 @@ use sp_runtime::{ generic::BlockId, }; use std::sync::Arc; -use sc_client_api::backend::Backend as ClientBackend; +use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; +use std::marker::PhantomData; /// params for block finalization. -pub struct FinalizeBlockParams { +pub struct FinalizeBlockParams { /// hash of the block pub hash: ::Hash, /// sender to report errors/success to the rpc. pub sender: rpc::Sender<()>, /// finalization justification pub justification: Option, - /// client backend - pub backend: Arc, + /// Finalizer trait object. + pub finalizer: Arc, + /// phantom type to pin the Backend type + pub _phantom: PhantomData, } + /// finalizes a block in the backend with the given params. -pub async fn finalize_block(params: FinalizeBlockParams) +pub async fn finalize_block(params: FinalizeBlockParams) where B: BlockT, + F: Finalizer, CB: ClientBackend, { let FinalizeBlockParams { hash, mut sender, justification, - backend: back_end, + finalizer, .. } = params; - match back_end.finalize_block(BlockId::Hash(hash), justification) { + match finalizer.finalize_block(BlockId::Hash(hash), justification, true) { Err(e) => { log::warn!("Failed to finalize block {:?}", e); rpc::send_result(&mut sender, Err(e.into())) } Ok(()) => { - log::info!("Successfully finalized block: {}", hash); + log::info!("✅ Successfully finalized block: {}", hash); rpc::send_result(&mut sender, Ok(())) } } -} \ No newline at end of file +} diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 36d12a004674487ec3be7c040c982f70d07d6a87..8294ae049f65808c4797afc9e34f2e6b423b6b39 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -17,66 +17,32 @@ //! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks. //! This is suitable for a testing environment. +use futures::prelude::*; use sp_consensus::{ - self, BlockImport, Environment, Proposer, BlockCheckParams, - ForkChoiceStrategy, BlockImportParams, BlockOrigin, - ImportResult, SelectChain, - import_queue::{ - BasicQueue, - CacheKeyId, - Verifier, - BoxBlockImport, - }, + Environment, Proposer, ForkChoiceStrategy, BlockImportParams, BlockOrigin, SelectChain, + import_queue::{BasicQueue, CacheKeyId, Verifier, BoxBlockImport}, }; +use sp_blockchain::HeaderBackend; use sp_inherents::InherentDataProviders; use sp_runtime::{traits::Block as BlockT, Justification}; -use sc_client_api::backend::Backend as ClientBackend; -use futures::prelude::*; +use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; use sc_transaction_pool::txpool; -use std::collections::HashMap; -use std::sync::Arc; +use std::{sync::Arc, marker::PhantomData}; -pub mod rpc; mod error; mod finalize_block; mod seal_new_block; -use finalize_block::{finalize_block, FinalizeBlockParams}; -use seal_new_block::{seal_new_block, SealBlockParams}; -pub use error::Error; -pub use rpc::{EngineCommand, CreatedBlock}; - -/// The synchronous block-import worker of the engine. -pub struct ManualSealBlockImport { - inner: I, -} - -impl From for ManualSealBlockImport { - fn from(i: I) -> Self { - ManualSealBlockImport { inner: i } - } -} - -impl BlockImport for ManualSealBlockImport - where - B: BlockT, - I: BlockImport, -{ - type Error = I::Error; - type Transaction = (); - - fn check_block(&mut self, block: BlockCheckParams) -> Result - { - self.inner.check_block(block) - } +pub mod rpc; - fn import_block( - &mut self, - block: BlockImportParams, - cache: HashMap>, - ) -> Result { - self.inner.import_block(block, cache) - } -} +use self::{ + finalize_block::{finalize_block, FinalizeBlockParams}, + seal_new_block::{seal_new_block, SealBlockParams}, +}; +pub use self::{ + error::Error, + rpc::{EngineCommand, CreatedBlock}, +}; +use sc_client_api::{TransactionFor, Backend}; /// The verifier for the manual seal engine; instantly finalizes. struct ManualSealVerifier; @@ -92,7 +58,7 @@ impl Verifier for ManualSealVerifier { let mut import_params = BlockImportParams::new(origin, header); import_params.justification = justification; import_params.body = body; - import_params.finalized = true; + import_params.finalized = false; import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain); Ok((import_params, None)) @@ -100,37 +66,43 @@ impl Verifier for ManualSealVerifier { } /// Instantiate the import queue for the manual seal consensus engine. -pub fn import_queue(block_import: BoxBlockImport) -> BasicQueue +pub fn import_queue( + block_import: BoxBlockImport> +) -> BasicQueue> + where + Block: BlockT, + B: Backend + 'static, { BasicQueue::new( ManualSealVerifier, - block_import, + Box::new(block_import), None, None, ) } /// Creates the background authorship task for the manual seal engine. -pub async fn run_manual_seal( +pub async fn run_manual_seal( mut block_import: BoxBlockImport, mut env: E, - backend: Arc, + client: Arc, pool: Arc>, - mut seal_block_channel: S, - select_chain: C, + mut commands_stream: S, + select_chain: SC, inherent_data_providers: InherentDataProviders, ) where + A: txpool::ChainApi::Hash> + 'static, B: BlockT + 'static, + C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, E: Environment + 'static, E::Error: std::fmt::Display, >::Error: std::fmt::Display, - A: txpool::ChainApi::Hash> + 'static, S: Stream::Hash>> + Unpin + 'static, - C: SelectChain + 'static, + SC: SelectChain + 'static, { - while let Some(command) = seal_block_channel.next().await { + while let Some(command) = commands_stream.next().await { match command { EngineCommand::SealNewBlock { create_empty, @@ -149,7 +121,7 @@ pub async fn run_manual_seal( block_import: &mut block_import, inherent_data_provider: &inherent_data_providers, pool: pool.clone(), - backend: backend.clone(), + client: client.clone(), } ).await; } @@ -159,7 +131,8 @@ pub async fn run_manual_seal( hash, sender, justification, - backend: backend.clone(), + finalizer: client.clone(), + _phantom: PhantomData, } ).await } @@ -170,26 +143,28 @@ pub async fn run_manual_seal( /// runs the background authorship task for the instant seal engine. /// instant-seal creates a new block for every transaction imported into /// the transaction pool. -pub async fn run_instant_seal( +pub async fn run_instant_seal( block_import: BoxBlockImport, env: E, - backend: Arc, + client: Arc, pool: Arc>, - select_chain: C, + select_chain: SC, inherent_data_providers: InherentDataProviders, ) where A: txpool::ChainApi::Hash> + 'static, B: BlockT + 'static, + C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, E: Environment + 'static, E::Error: std::fmt::Display, >::Error: std::fmt::Display, - C: SelectChain + 'static + SC: SelectChain + 'static { // instant-seal creates blocks as soon as transactions are imported // into the transaction pool. - let seal_block_channel = pool.validated_pool().import_notification_stream() + let commands_stream = pool.validated_pool() + .import_notification_stream() .map(|_| { EngineCommand::SealNewBlock { create_empty: false, @@ -202,9 +177,9 @@ pub async fn run_instant_seal( run_manual_seal( block_import, env, - backend, + client, pool, - seal_block_channel, + commands_stream, select_chain, inherent_data_providers, ).await @@ -224,11 +199,9 @@ mod tests { txpool::Options, }; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; - use sp_transaction_pool::{TransactionPool, MaintainedTransactionPool}; + use sp_transaction_pool::{TransactionPool, MaintainedTransactionPool, TransactionSource}; use sp_runtime::generic::BlockId; - use sp_blockchain::HeaderBackend; use sp_consensus::ImportedAux; - use sc_client::LongestChain; use sp_inherents::InherentDataProviders; use sc_basic_authorship::ProposerFactory; @@ -236,12 +209,13 @@ mod tests { Arc::new(TestApi::empty()) } + const SOURCE: TransactionSource = TransactionSource::External; + #[tokio::test] async fn instant_seal() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); let pool = Arc::new(BasicPool::new(Options::default(), api()).0); let env = ProposerFactory::new( @@ -266,7 +240,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -278,7 +252,7 @@ mod tests { rt.block_on(future); }); // submit a transaction to pool. - let result = pool.submit_one(&BlockId::Number(0), uxt(Alice, 0)).await; + let result = pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Alice, 0)).await; // assert that it was successfully imported assert!(result.is_ok()); // assert that the background task returns ok @@ -298,15 +272,14 @@ mod tests { } ); // assert that there's a new block in the db. - assert!(backend.blockchain().header(BlockId::Number(1)).unwrap().is_some()) + assert!(client.header(&BlockId::Number(1)).unwrap().is_some()) } #[tokio::test] async fn manual_seal_and_finalization() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); let pool = Arc::new(BasicPool::new(Options::default(), api()).0); let env = ProposerFactory::new( @@ -318,7 +291,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -330,7 +303,7 @@ mod tests { rt.block_on(future); }); // submit a transaction to pool. - let result = pool.submit_one(&BlockId::Number(0), uxt(Alice, 0)).await; + let result = pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Alice, 0)).await; // assert that it was successfully imported assert!(result.is_ok()); let (tx, rx) = futures::channel::oneshot::channel(); @@ -358,7 +331,7 @@ mod tests { } ); // assert that there's a new block in the db. - let header = backend.blockchain().header(BlockId::Number(1)).unwrap().unwrap(); + let header = client.header(&BlockId::Number(1)).unwrap().unwrap(); let (tx, rx) = futures::channel::oneshot::channel(); sink.send(EngineCommand::FinalizeBlock { sender: Some(tx), @@ -372,9 +345,8 @@ mod tests { #[tokio::test] async fn manual_seal_fork_blocks() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); let pool_api = api(); let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone()).0); @@ -387,7 +359,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -399,7 +371,7 @@ mod tests { rt.block_on(future); }); // submit a transaction to pool. - let result = pool.submit_one(&BlockId::Number(0), uxt(Alice, 0)).await; + let result = pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Alice, 0)).await; // assert that it was successfully imported assert!(result.is_ok()); @@ -429,12 +401,12 @@ mod tests { } ); // assert that there's a new block in the db. - assert!(backend.blockchain().header(BlockId::Number(0)).unwrap().is_some()); - assert!(pool.submit_one(&BlockId::Number(1), uxt(Alice, 1)).await.is_ok()); + assert!(client.header(&BlockId::Number(0)).unwrap().is_some()); + assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Alice, 1)).await.is_ok()); pool.maintain(sp_transaction_pool::ChainEvent::NewBlock { id: BlockId::Number(1), - header: backend.blockchain().header(BlockId::Number(1)).expect("db error").expect("imported above"), + header: client.header(&BlockId::Number(1)).expect("db error").expect("imported above"), is_new_best: true, retracted: vec![], }).await; @@ -450,10 +422,10 @@ mod tests { rx1.await.expect("should be no error receiving"), Ok(_) ); - assert!(backend.blockchain().header(BlockId::Number(1)).unwrap().is_some()); + assert!(client.header(&BlockId::Number(1)).unwrap().is_some()); pool_api.increment_nonce(Alice.into()); - assert!(pool.submit_one(&BlockId::Number(2), uxt(Alice, 2)).await.is_ok()); + assert!(pool.submit_one(&BlockId::Number(2), SOURCE, uxt(Alice, 2)).await.is_ok()); let (tx2, rx2) = futures::channel::oneshot::channel(); assert!(sink.send(EngineCommand::SealNewBlock { parent_hash: Some(created_block.hash), @@ -463,6 +435,6 @@ mod tests { }).await.is_ok()); let imported = rx2.await.unwrap().unwrap(); // assert that fork block is in the db - assert!(backend.blockchain().header(BlockId::Hash(imported.hash)).unwrap().is_some()) + assert!(client.header(&BlockId::Hash(imported.hash)).unwrap().is_some()) } } diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_new_block.rs index 39d73e16ab74fdadde2e3a597fa2ef71615cc9fa..88b58ef4cc2b39cbbe3af1209e54408390e074ef 100644 --- a/client/consensus/manual-seal/src/seal_new_block.rs +++ b/client/consensus/manual-seal/src/seal_new_block.rs @@ -33,7 +33,6 @@ use sp_consensus::{ import_queue::BoxBlockImport, }; use sp_blockchain::HeaderBackend; -use sc_client_api::backend::Backend as ClientBackend; use std::collections::HashMap; use std::time::Duration; use sp_inherents::InherentDataProviders; @@ -42,7 +41,7 @@ use sp_inherents::InherentDataProviders; const MAX_PROPOSAL_DURATION: u64 = 10; /// params for sealing a new block -pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { +pub struct SealBlockParams<'a, B: BlockT, SC, HB, E, T, P: txpool::ChainApi> { /// if true, empty blocks(without extrinsics) will be created. /// otherwise, will return Error::EmptyTransactionPool. pub create_empty: bool, @@ -54,12 +53,12 @@ pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { pub sender: rpc::Sender::Hash>>, /// transaction pool pub pool: Arc>, - /// client backend - pub backend: Arc, + /// header backend + pub client: Arc, /// Environment trait object for creating a proposer pub env: &'a mut E, /// SelectChain object - pub select_chain: &'a C, + pub select_chain: &'a SC, /// block import object pub block_import: &'a mut BoxBlockImport, /// inherent data provider @@ -67,24 +66,24 @@ pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { } /// seals a new block with the given params -pub async fn seal_new_block( +pub async fn seal_new_block( SealBlockParams { create_empty, finalize, pool, parent_hash, - backend: back_end, + client, select_chain, block_import, env, inherent_data_provider, mut sender, .. - }: SealBlockParams<'_, B, SC, CB, E, T, P> + }: SealBlockParams<'_, B, SC, HB, E, T, P> ) where B: BlockT, - CB: ClientBackend, + HB: HeaderBackend, E: Environment, >::Error: std::fmt::Display, >::Error: std::fmt::Display, @@ -101,7 +100,7 @@ pub async fn seal_new_block( // or fetch the best_block. let header = match parent_hash { Some(hash) => { - match back_end.blockchain().header(BlockId::Hash(hash))? { + match client.header(BlockId::Hash(hash))? { Some(header) => header, None => return Err(Error::BlockNotFound(format!("{}", hash))), } diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index a168c0b380625482bc753ec9a8fd9cf11ef46742..c7832baae0ee4a2a27a35b88c4ec34f31e0854e5 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -9,7 +9,7 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } @@ -23,3 +23,6 @@ log = "0.4.8" futures = { version = "0.3.1", features = ["compat"] } sp-timestamp = { version = "2.0.0-alpha.5", path = "../../../primitives/timestamp" } derive_more = "0.99.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index 49d2e64f6050c92b8603e6a3cf034e19263085a1..de41ea7bd2356f02f5857344b70eae335273a540 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -151,7 +151,7 @@ pub trait PowAlgorithm { /// /// This function will be called twice during the import process, so the implementation /// should be properly cached. - fn difficulty(&self, parent: &BlockId) -> Result>; + fn difficulty(&self, parent: B::Hash) -> Result>; /// Verify that the seal is valid against given pre hash when parent block is not yet imported. /// /// None means that preliminary verify is not available for this algorithm. @@ -335,7 +335,7 @@ impl BlockImport for PowBlockImport difficulty, - None => self.algorithm.difficulty(&BlockId::hash(parent_hash))?, + None => self.algorithm.difficulty(parent_hash)?, }; let pre_hash = block.header.hash(); @@ -617,9 +617,7 @@ fn mine_loop( let (header, body) = proposal.block.deconstruct(); let (difficulty, seal) = { - let difficulty = algorithm.difficulty( - &BlockId::Hash(best_hash), - )?; + let difficulty = algorithm.difficulty(best_hash)?; loop { let seal = algorithm.mine( diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index fd05cf7ca4e4a065b8d6a238dec4716e2cad4433..bf973ef47a893bbd45a6bb47b03a9f5afbfd99df 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -10,7 +10,7 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } @@ -20,10 +20,13 @@ sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } sc-telemetry = { version = "2.0.0-alpha.5", path = "../../telemetry" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index 7fc4bdc7d27e483f8586e16150b348a59153d692..5952856bdafd960b06bb6d71b197b0f9982c3c55 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -258,10 +258,10 @@ pub trait SimpleSlotWorker { Box::new(futures::future::select(proposing, delay).map(move |v| match v { futures::future::Either::Left((b, _)) => b.map(|b| (b, claim)), futures::future::Either::Right(_) => { - info!("Discarding proposal for slot {}; block production took too long", slot_number); + info!("⌛️ Discarding proposal for slot {}; block production took too long", slot_number); // If the node was compiled with debug, tell the user to use release optimizations. #[cfg(build_type="debug")] - info!("Recompile your node in `--release` mode to mitigate this problem."); + info!("👉 Recompile your node in `--release` mode to mitigate this problem."); telemetry!(CONSENSUS_INFO; "slots.discarding_proposal_took_too_long"; "slot" => slot_number, ); @@ -289,7 +289,7 @@ pub trait SimpleSlotWorker { ); info!( - "Pre-sealed block for proposal at {}. Hash now {:?}, previously {:?}.", + "🔖 Pre-sealed block for proposal at {}. Hash now {:?}, previously {:?}.", header_num, block_import_params.post_hash(), header_hash, @@ -466,7 +466,7 @@ impl SlotDuration { cb(client.runtime_api(), &BlockId::number(Zero::zero()))?; info!( - "Loaded block-time = {:?} milliseconds from genesis on first-launch", + "⏱ Loaded block-time = {:?} milliseconds from genesis on first-launch", genesis_slot_duration ); diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index 92d36f0bfd9823d157c277ff4111f1fdcd49c9d8..7e8014199baa5241935cc198cce100bba9677cfa 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -16,3 +16,6 @@ sp-authorship = { version = "2.0.0-alpha.5", path = "../../../primitives/authors sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } log = "0.4.8" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index b2ece9b7ff6603231780e5e53a2c340d7f54e4fa..9308d4ee74adf169deb8faff092fd81f26cfbdd9 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -18,7 +18,7 @@ kvdb-memorydb = "0.5.0" linked-hash-map = "0.5.2" hash-db = "0.15.2" parity-util-mem = { version = "0.6.0", default-features = false, features = ["std"] } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } @@ -43,3 +43,6 @@ tempfile = "3" [features] default = [] test-helpers = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 05ec51f1c8d383bdbedb5ac346182235161300c3..02b30a085af2b9fddf3e22671d3f3ce0c469ec0a 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use std::path::PathBuf; use std::cell::{Cell, RefCell}; +use std::collections::HashMap; use rand::Rng; use hash_db::{Prefix, Hasher}; @@ -54,12 +55,14 @@ pub struct BenchmarkingState { genesis_root: B::Hash, state: RefCell>>, db: Cell>>, - genesis: as StateBackend>>::Transaction, + genesis: HashMap, (Vec, i32)>, + record: Cell>>, + cache_size_mb: Option, } impl BenchmarkingState { /// Create a new instance that creates a database in a temporary dir. - pub fn new(genesis: Storage) -> Result { + pub fn new(genesis: Storage, cache_size_mb: Option) -> Result { let temp_dir = PathBuf::from(std::env::temp_dir()); let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect(); let path = temp_dir.join(&name); @@ -76,6 +79,8 @@ impl BenchmarkingState { root: Cell::new(root), genesis: Default::default(), genesis_root: Default::default(), + record: Default::default(), + cache_size_mb, }; state.reopen()?; @@ -88,7 +93,7 @@ impl BenchmarkingState { genesis.top.into_iter().map(|(k, v)| (k, Some(v))), child_delta, ); - state.genesis = transaction.clone(); + state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); state.commit(root, transaction)?; Ok(state) @@ -97,7 +102,10 @@ impl BenchmarkingState { fn reopen(&self) -> Result<(), String> { *self.state.borrow_mut() = None; self.db.set(None); - let db_config = DatabaseConfig::with_columns(1); + let mut db_config = DatabaseConfig::with_columns(1); + if let Some(size) = &self.cache_size_mb { + db_config.memory_budget.insert(0, *size); + } let path = self.path.to_str() .ok_or_else(|| String::from("Invalid database path"))?; let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?); @@ -257,14 +265,17 @@ impl StateBackend> for BenchmarkingState { { if let Some(db) = self.db.take() { let mut db_transaction = DBTransaction::new(); - - for (key, (val, rc)) in transaction.drain() { + let changes = transaction.drain(); + let mut keys = Vec::with_capacity(changes.len()); + for (key, (val, rc)) in changes { if rc > 0 { db_transaction.put(0, &key, &val); } else if rc < 0 { db_transaction.delete(0, &key); } + keys.push(key); } + self.record.set(keys); db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; self.root.set(storage_root); } else { @@ -274,11 +285,34 @@ impl StateBackend> for BenchmarkingState { } fn wipe(&self) -> Result<(), Self::Error> { - self.kill()?; + // Restore to genesis + let record = self.record.take(); + if let Some(db) = self.db.take() { + let mut db_transaction = DBTransaction::new(); + for key in record { + match self.genesis.get(&key) { + Some((v, _)) => db_transaction.put(0, &key, v), + None => db_transaction.delete(0, &key), + } + } + db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; + } + + self.db.set(None); + *self.state.borrow_mut() = None; + + self.root.set(self.genesis_root.clone()); self.reopen()?; - self.commit(self.genesis_root.clone(), self.genesis.clone())?; Ok(()) } + + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { + self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats)); + } + + fn usage_info(&self) -> sp_state_machine::UsageInfo { + self.state.borrow().as_ref().map_or(sp_state_machine::UsageInfo::empty(), |s| s.usage_info()) + } } impl std::fmt::Debug for BenchmarkingState { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 62a8a5b43e037a34c0f3f48620a597789308b795..e00d608a5feed9546bb9591194d46e2f350d72ab 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -73,7 +73,7 @@ use sc_executor::RuntimeInfo; use sp_state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, UsageInfo as StateUsageInfo, StorageCollection, ChildStorageCollection, - backend::Backend as StateBackend, + backend::Backend as StateBackend, StateMachineStats, }; use crate::utils::{DatabaseType, Meta, db_err, meta_keys, read_db, read_meta}; use crate::changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTransaction}; @@ -256,6 +256,14 @@ impl StateBackend> for RefTrackingState { { self.state.as_trie_backend() } + + fn register_overlay_stats(&mut self, stats: &StateMachineStats) { + self.state.register_overlay_stats(stats); + } + + fn usage_info(&self) -> StateUsageInfo { + self.state.usage_info() + } } /// Database settings. @@ -276,8 +284,8 @@ pub enum DatabaseSettingsSrc { Path { /// Path to the database. path: PathBuf, - /// Cache size in bytes. If `None` default is used. - cache_size: Option, + /// Cache size in bytes. + cache_size: usize, }, /// Use a custom already-open database. @@ -417,6 +425,7 @@ impl sc_client::blockchain::HeaderBackend for BlockchainDb genesis_hash: meta.genesis_hash, finalized_hash: meta.finalized_hash, finalized_number: meta.finalized_number, + number_leaves: self.leaves.read().count(), } } @@ -1116,6 +1125,8 @@ impl Backend { let mut changeset: sc_state_db::ChangeSet> = sc_state_db::ChangeSet::default(); let mut ops: u64 = 0; let mut bytes: u64 = 0; + let mut removal: u64 = 0; + let mut bytes_removal: u64 = 0; for (key, (val, rc)) in operation.db_updates.drain() { if rc > 0 { ops += 1; @@ -1123,14 +1134,26 @@ impl Backend { changeset.inserted.push((key, val.to_vec())); } else if rc < 0 { - ops += 1; - bytes += key.len() as u64; + removal += 1; + bytes_removal += key.len() as u64; changeset.deleted.push(key); } } - self.state_usage.tally_writes(ops, bytes); + self.state_usage.tally_writes_nodes(ops, bytes); + self.state_usage.tally_removed_nodes(removal, bytes_removal); + let mut ops: u64 = 0; + let mut bytes: u64 = 0; + for (key, value) in operation.storage_updates.iter() + .chain(operation.child_storage_updates.iter().flat_map(|(_, s)| s.iter())) { + ops += 1; + bytes += key.len() as u64; + if let Some(v) = value.as_ref() { + bytes += v.len() as u64; + } + } + self.state_usage.tally_writes(ops, bytes); let number_u64 = number.saturated_into::(); let commit = self.storage.state_db.insert_block( &hash, @@ -1498,8 +1521,10 @@ impl sc_client_api::backend::Backend for Backend { reads: io_stats.reads, average_transaction_size: io_stats.avg_transaction_size() as u64, state_reads: state_stats.reads.ops, - state_reads_cache: state_stats.cache_reads.ops, state_writes: state_stats.writes.ops, + state_writes_cache: state_stats.overlay_writes.ops, + state_reads_cache: state_stats.cache_reads.ops, + state_writes_nodes: state_stats.nodes_writes.ops, }, }) } diff --git a/client/db/src/light.rs b/client/db/src/light.rs index cda1a1195268e74893b6a18440f23d7d1495d971..e3dcdedd5096d533dbe4c0da4452e47161e3b866 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -153,6 +153,7 @@ impl BlockchainHeaderBackend for LightStorage genesis_hash: meta.genesis_hash, finalized_hash: meta.finalized_hash, finalized_number: meta.finalized_number, + number_leaves: 1, } } @@ -592,8 +593,10 @@ impl LightBlockchainStorage for LightStorage average_transaction_size: io_stats.avg_transaction_size() as u64, // Light client does not track those state_reads: 0, - state_reads_cache: 0, state_writes: 0, + state_reads_cache: 0, + state_writes_cache: 0, + state_writes_nodes: 0, } }) } @@ -960,7 +963,7 @@ pub(crate) mod tests { fn run_checks(db: &LightStorage, max: u64, checks: &[(u64, Option>)]) { for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) { - let actual = get_authorities(db.cache(), BlockId::Number(*at)); + let actual = authorities(db.cache(), BlockId::Number(*at)); assert_eq!(*expected, actual); } } @@ -975,7 +978,7 @@ pub(crate) mod tests { map } - fn get_authorities(cache: &dyn BlockchainCache, at: BlockId) -> Option> { + fn authorities(cache: &dyn BlockchainCache, at: BlockId) -> Option> { cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None) .and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok()) } @@ -1023,9 +1026,9 @@ pub(crate) mod tests { // ... -> B2(1) -> B2_1(1) -> B2_2(2) // => the cache ignores all writes before best finalized block let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3)); - assert_eq!(None, get_authorities(db.cache(), BlockId::Hash(hash2_1))); + assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_1))); let hash2_2 = insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash2_1, 4)); - assert_eq!(None, get_authorities(db.cache(), BlockId::Hash(hash2_2))); + assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_2))); } let (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) = { @@ -1037,55 +1040,55 @@ pub(crate) mod tests { let hash7 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); let hash8 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); let hash6_1 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); let hash6_2 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) }; @@ -1094,27 +1097,27 @@ pub(crate) mod tests { // finalize block hash6_1 db.finalize_header(BlockId::Hash(hash6_1)).unwrap(); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); // finalize block hash6_2 db.finalize_header(BlockId::Hash(hash6_2)).unwrap(); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); } } diff --git a/client/db/src/stats.rs b/client/db/src/stats.rs index 1d6ed8e7f0493c9adf7d516bfacd0467c231dae7..8bc93b5b644cf985ff81fa0aaa0c0154c64eeaa0 100644 --- a/client/db/src/stats.rs +++ b/client/db/src/stats.rs @@ -25,6 +25,10 @@ pub struct StateUsageStats { bytes_read: AtomicU64, writes: AtomicU64, bytes_written: AtomicU64, + writes_nodes: AtomicU64, + bytes_written_nodes: AtomicU64, + removed_nodes: AtomicU64, + bytes_removed_nodes: AtomicU64, reads_cache: AtomicU64, bytes_read_cache: AtomicU64, } @@ -38,6 +42,10 @@ impl StateUsageStats { bytes_read: 0.into(), writes: 0.into(), bytes_written: 0.into(), + writes_nodes: 0.into(), + bytes_written_nodes: 0.into(), + removed_nodes: 0.into(), + bytes_removed_nodes: 0.into(), reads_cache: 0.into(), bytes_read_cache: 0.into(), } @@ -70,7 +78,19 @@ impl StateUsageStats { val } - /// Tally some write operations, including their byte count. + /// Tally some write trie nodes operations, including their byte count. + pub fn tally_writes_nodes(&self, ops: u64, data_bytes: u64) { + self.writes_nodes.fetch_add(ops, AtomicOrdering::Relaxed); + self.bytes_written_nodes.fetch_add(data_bytes, AtomicOrdering::Relaxed); + } + + /// Tally some removed trie nodes operations, including their byte count. + pub fn tally_removed_nodes(&self, ops: u64, data_bytes: u64) { + self.removed_nodes.fetch_add(ops, AtomicOrdering::Relaxed); + self.bytes_removed_nodes.fetch_add(data_bytes, AtomicOrdering::Relaxed); + } + + /// Tally some write trie nodes operations, including their byte count. pub fn tally_writes(&self, ops: u64, data_bytes: u64) { self.writes.fetch_add(ops, AtomicOrdering::Relaxed); self.bytes_written.fetch_add(data_bytes, AtomicOrdering::Relaxed); @@ -80,8 +100,10 @@ impl StateUsageStats { pub fn merge_sm(&self, info: sp_state_machine::UsageInfo) { self.reads.fetch_add(info.reads.ops, AtomicOrdering::Relaxed); self.bytes_read.fetch_add(info.reads.bytes, AtomicOrdering::Relaxed); - self.writes.fetch_add(info.writes.ops, AtomicOrdering::Relaxed); - self.bytes_written.fetch_add(info.writes.bytes, AtomicOrdering::Relaxed); + self.writes_nodes.fetch_add(info.nodes_writes.ops, AtomicOrdering::Relaxed); + self.bytes_written_nodes.fetch_add(info.nodes_writes.bytes, AtomicOrdering::Relaxed); + self.removed_nodes.fetch_add(info.removed_nodes.ops, AtomicOrdering::Relaxed); + self.bytes_removed_nodes.fetch_add(info.removed_nodes.bytes, AtomicOrdering::Relaxed); self.reads_cache.fetch_add(info.cache_reads.ops, AtomicOrdering::Relaxed); self.bytes_read_cache.fetch_add(info.cache_reads.bytes, AtomicOrdering::Relaxed); } @@ -100,7 +122,11 @@ impl StateUsageStats { sp_state_machine::UsageInfo { reads: unit(&self.reads, &self.bytes_read), writes: unit(&self.writes, &self.bytes_written), + nodes_writes: unit(&self.writes_nodes, &self.bytes_written_nodes), + removed_nodes: unit(&self.removed_nodes, &self.bytes_removed_nodes), cache_reads: unit(&self.reads_cache, &self.bytes_read_cache), + modified_reads: Default::default(), + overlay_writes: Default::default(), // TODO: Proper tracking state of memory footprint here requires // imposing `MallocSizeOf` requirement on half of the codebase, // so it is an open question how to do it better diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 2ac1ee3dbd5f4af37f6fd21d72a55ee7a5748d1d..63268992632410934d0518039882240dee328c0d 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -299,6 +299,8 @@ pub struct CacheChanges { pub struct CachingState { /// Usage statistics usage: StateUsageStats, + /// State machine registered stats + overlay_stats: sp_state_machine::StateMachineStats, /// Backing state. state: S, /// Cache data. @@ -428,6 +430,7 @@ impl>, B: BlockT> CachingState { ) -> Self { CachingState { usage: StateUsageStats::new(), + overlay_stats: sp_state_machine::StateMachineStats::default(), state, cache: CacheChanges { shared_cache, @@ -663,8 +666,14 @@ impl>, B: BlockT> StateBackend> for Cachin self.state.as_trie_backend() } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { + self.overlay_stats.add(stats); + } + fn usage_info(&self) -> sp_state_machine::UsageInfo { - self.usage.take() + let mut info = self.usage.take(); + info.include_state_machine_states(&self.overlay_stats); + info } } @@ -852,6 +861,10 @@ impl>, B: BlockT> StateBackend> for Syncin .as_trie_backend() } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { + self.caching_state().register_overlay_stats(stats); + } + fn usage_info(&self) -> sp_state_machine::UsageInfo { self.caching_state().usage_info() } diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 971acf8456b6463fe315768f16e2f8cf3af8254b..5e6794108ecfe87a1928f83a7939f81acac6d2e2 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -166,7 +166,7 @@ mod tests { state_cache_size: 0, state_cache_child_ratio: None, pruning: PruningMode::ArchiveAll, - source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None }, + source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: 128 }, }, DatabaseType::Full).map(|_| ()) } diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index f26714eb5a7dbf9edae0fcac6a850e877478fd0e..e62c4f052bb26573b5b16c7a88f2db5c69ab65f2 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -227,24 +227,22 @@ pub fn open_database( // and now open database assuming that it has the latest version let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + let state_col_budget = (*cache_size as f64 * 0.9) as usize; + let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1); + let mut memory_budget = std::collections::HashMap::new(); + let path = path.to_str() + .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; - if let Some(cache_size) = cache_size { - let state_col_budget = (*cache_size as f64 * 0.9) as usize; - let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1); - - let mut memory_budget = std::collections::HashMap::new(); - for i in 0..NUM_COLUMNS { - if i == crate::columns::STATE { - memory_budget.insert(i, state_col_budget); - } else { - memory_budget.insert(i, other_col_budget); - } + for i in 0..NUM_COLUMNS { + if i == crate::columns::STATE { + memory_budget.insert(i, state_col_budget); + } else { + memory_budget.insert(i, other_col_budget); } - - db_config.memory_budget = memory_budget; } - let path = path.to_str() - .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; + + db_config.memory_budget = memory_budget; + Arc::new(Database::open(&db_config, &path).map_err(db_err)?) }, #[cfg(not(any(feature = "kvdb-rocksdb", test)))] diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index 4dca9fa6d539f8ae4b74a32ab9f61cb8fd1edb88..cae0d56d8ed5eea8fef0e77c92eec6c2d5d9a61b 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/sc-executor" [dependencies] derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } @@ -52,3 +52,6 @@ wasmtime = [ wasmi-errno = [ "wasmi/errno" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index 14a1f8333b558c6ba1bdd0ba0e1cf74750250db7..1f52b959d00b1fc72d56b112267a4d70eb0d1b90 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/sc-executor-common/" [dependencies] log = "0.4.8" derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } wasmi = "0.6.2" sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-allocator = { version = "2.0.0-alpha.5", path = "../../../primitives/allocator" } @@ -22,3 +22,6 @@ sp-serializer = { version = "2.0.0-alpha.5", path = "../../../primitives/seriali [features] default = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 7b75992234fd8c7cd561b8943d1dea66e38c1f30..e50061f4f24bb71639a5204cf326d6be7d397e6f 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -28,3 +28,6 @@ std = [ "sp-std/std", "sp-allocator/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index 5641755ef18ba10b1312faba4ec3730343483935..778bc808004bb2470b528c802bd14c1d0fe1bc51 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -483,5 +483,7 @@ mod tests { 2, ); }); + + my_interface::say_hello_world("hey"); } } diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index e6c16453f365fb560f577bcda600edbbb586845a..7a369cc470ab55c4413dc3c3cc3beb7604a45068 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -40,6 +40,12 @@ pub enum WasmExecutionMethod { Compiled, } +impl Default for WasmExecutionMethod { + fn default() -> WasmExecutionMethod { + WasmExecutionMethod::Interpreted + } +} + /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { /// Runtime code hash. diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index bae36353a2dfa2468920ce91ec051086fd8be16b..ea8637b9e2830972b19639440546cf486b606c5d 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -13,9 +13,12 @@ documentation = "https://docs.rs/sc-executor-wasmi" log = "0.4.8" wasmi = "0.6.2" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sc-executor-common = { version = "0.8.0-alpha.5", path = "../common" } sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/wasm-interface" } sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime-interface" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-allocator = { version = "2.0.0-alpha.5", path = "../../../primitives/allocator" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 3b30e63fbb334b6161a4005984a9e142397e0f73..11f99e7876bbc57dae3b3ec792a3efe30161d476 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -12,7 +12,7 @@ description = "Defines a `WasmRuntime` that uses the Wasmtime JIT to execute." log = "0.4.8" scoped-tls = "1.0" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sc-executor-common = { version = "0.8.0-alpha.5", path = "../common" } sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/wasm-interface" } sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime-interface" } @@ -22,3 +22,6 @@ wasmtime = { package = "substrate-wasmtime", version = "0.13.0-threadsafe.1" } [dev-dependencies] assert_matches = "1.3.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index a202f9d20541f9595dc36ddbc848254edf30782e..b684c814d1b93607534c2b73fdc74aa4a99eda8b 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -12,15 +12,16 @@ documentation = "https://docs.rs/sc-finality-grandpa" [dependencies] fork-tree = { version = "2.0.0-alpha.5", path = "../../utils/fork-tree" } -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" assert_matches = "1.3.0" -parity-scale-codec = { version = "1.2.0", features = ["derive"] } +parity-scale-codec = { version = "1.3.0", features = ["derive"] } sp-arithmetic = { version = "2.0.0-alpha.5", path = "../../primitives/arithmetic" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } @@ -52,3 +53,6 @@ env_logger = "0.7.0" tokio = { version = "0.2", features = ["rt-core"] } tempfile = "3.1.0" sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/finality-grandpa/src/authorities.rs b/client/finality-grandpa/src/authorities.rs index 5e295d0baee7ab86548a1e6878090d85d885efad..fe3f2dd19eb1ce5764508fe5ff86e5919d35a2e7 100644 --- a/client/finality-grandpa/src/authorities.rs +++ b/client/finality-grandpa/src/authorities.rs @@ -20,7 +20,7 @@ use fork_tree::ForkTree; use parking_lot::RwLock; use finality_grandpa::voter_set::VoterSet; use parity_scale_codec::{Encode, Decode}; -use log::{debug, info}; +use log::debug; use sc_telemetry::{telemetry, CONSENSUS_INFO}; use sp_finality_grandpa::{AuthorityId, AuthorityList}; @@ -250,6 +250,7 @@ where best_hash: H, best_number: N, is_descendent_of: &F, + initial_sync: bool, ) -> Result, E> where F: Fn(&H, &H) -> Result, { @@ -262,8 +263,10 @@ where // check if the given best block is in the same branch as the block that signaled the change. if is_descendent_of(&change.canon_hash, &best_hash)? { // apply this change: make the set canonical - info!(target: "afg", "Applying authority set change forced at block #{:?}", - change.canon_height); + afg_log!(initial_sync, + "👴 Applying authority set change forced at block #{:?}", + change.canon_height, + ); telemetry!(CONSENSUS_INFO; "afg.applying_forced_authority_set_change"; "block" => ?change.canon_height ); @@ -305,6 +308,7 @@ where finalized_hash: H, finalized_number: N, is_descendent_of: &F, + initial_sync: bool, ) -> Result, fork_tree::Error> where F: Fn(&H, &H) -> Result, E: std::error::Error, @@ -328,8 +332,10 @@ where self.pending_forced_changes.clear(); if let Some(change) = change { - info!(target: "afg", "Applying authority set change scheduled at block #{:?}", - change.canon_height); + afg_log!(initial_sync, + "👴 Applying authority set change scheduled at block #{:?}", + change.canon_height, + ); telemetry!(CONSENSUS_INFO; "afg.applying_scheduled_authority_set_change"; "block" => ?change.canon_height ); @@ -599,11 +605,16 @@ mod tests { ); // finalizing "hash_c" won't enact the change signaled at "hash_a" but it will prune out "hash_b" - let status = authorities.apply_standard_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) { - ("hash_a", "hash_c") => true, - ("hash_b", "hash_c") => false, - _ => unreachable!(), - })).unwrap(); + let status = authorities.apply_standard_changes( + "hash_c", + 11, + &is_descendent_of(|base, hash| match (*base, *hash) { + ("hash_a", "hash_c") => true, + ("hash_b", "hash_c") => false, + _ => unreachable!(), + }), + false, + ).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, None); @@ -613,10 +624,15 @@ mod tests { ); // finalizing "hash_d" will enact the change signaled at "hash_a" - let status = authorities.apply_standard_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) { - ("hash_a", "hash_d") => true, - _ => unreachable!(), - })).unwrap(); + let status = authorities.apply_standard_changes( + "hash_d", + 15, + &is_descendent_of(|base, hash| match (*base, *hash) { + ("hash_a", "hash_d") => true, + _ => unreachable!(), + }), + false, + ).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_d", 15))); @@ -671,12 +687,18 @@ mod tests { }); // trying to finalize past `change_c` without finalizing `change_a` first - match authorities.apply_standard_changes("hash_d", 40, &is_descendent_of) { - Err(fork_tree::Error::UnfinalizedAncestor) => {}, - _ => unreachable!(), - } + assert!(matches!( + authorities.apply_standard_changes("hash_d", 40, &is_descendent_of, false), + Err(fork_tree::Error::UnfinalizedAncestor) + )); + + let status = authorities.apply_standard_changes( + "hash_b", + 15, + &is_descendent_of, + false, + ).unwrap(); - let status = authorities.apply_standard_changes("hash_b", 15, &is_descendent_of).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_b", 15))); @@ -684,7 +706,13 @@ mod tests { assert_eq!(authorities.set_id, 1); // after finalizing `change_a` it should be possible to finalize `change_c` - let status = authorities.apply_standard_changes("hash_d", 40, &is_descendent_of).unwrap(); + let status = authorities.apply_standard_changes( + "hash_d", + 40, + &is_descendent_of, + false, + ).unwrap(); + assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_d", 40))); @@ -817,20 +845,30 @@ mod tests { assert!(authorities.add_pending_change(change_c, &is_descendent_of_a).is_err()); // too early. - assert!(authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true)).unwrap().is_none()); + assert!( + authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true), false) + .unwrap() + .is_none() + ); // too late. - assert!(authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true)).unwrap().is_none()); + assert!( + authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true), false) + .unwrap() + .is_none() + ); // on time -- chooses the right change. assert_eq!( - authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a).unwrap().unwrap(), + authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a, false) + .unwrap() + .unwrap(), (42, AuthoritySet { current_authorities: set_a, set_id: 1, pending_standard_changes: ForkTree::new(), pending_forced_changes: Vec::new(), - }) + }), ); } } diff --git a/client/finality-grandpa/src/aux_schema.rs b/client/finality-grandpa/src/aux_schema.rs index 525a4a99bab5861e57b6fc25b49bbcaccaea7220..fe652f52fe2067e728263efe0acd302d2673be2b 100644 --- a/client/finality-grandpa/src/aux_schema.rs +++ b/client/finality-grandpa/src/aux_schema.rs @@ -330,7 +330,7 @@ pub(crate) fn load_persistent( } // genesis. - info!(target: "afg", "Loading GRANDPA authority set \ + info!(target: "afg", "👴 Loading GRANDPA authority set \ from genesis on what appears to be first startup."); let genesis_authorities = genesis_authorities()?; diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index 6b245a4652a5f0833417ad13b3e6d566768bd882..2d39ed7ec433fa2fc9e4f931e1916e922328137d 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -84,13 +84,13 @@ use sp_runtime::traits::{NumberFor, Block as BlockT, Zero}; use sc_network_gossip::{MessageIntent, ValidatorContext}; -use sc_network::{config::Roles, PeerId, ReputationChange}; +use sc_network::{ObservedRole, PeerId, ReputationChange}; use parity_scale_codec::{Encode, Decode}; use sp_finality_grandpa::AuthorityId; use sc_telemetry::{telemetry, CONSENSUS_DEBUG}; use log::{trace, debug}; -use futures::channel::mpsc; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use prometheus_endpoint::{CounterVec, Opts, PrometheusError, register, Registry, U64}; use rand::seq::SliceRandom; @@ -439,11 +439,11 @@ impl Misbehavior { struct PeerInfo { view: View, - roles: Roles, + roles: ObservedRole, } impl PeerInfo { - fn new(roles: Roles) -> Self { + fn new(roles: ObservedRole) -> Self { PeerInfo { view: View::default(), roles, @@ -469,14 +469,17 @@ impl Default for Peers { } impl Peers { - fn new_peer(&mut self, who: PeerId, roles: Roles) { - if roles.is_authority() && self.lucky_authorities.len() < MIN_LUCKY { - self.lucky_authorities.insert(who.clone()); - } - if !roles.is_authority() && self.lucky_peers.len() < MIN_LUCKY { - self.lucky_peers.insert(who.clone()); + fn new_peer(&mut self, who: PeerId, role: ObservedRole) { + match role { + ObservedRole::Authority if self.lucky_authorities.len() < MIN_LUCKY => { + self.lucky_authorities.insert(who.clone()); + }, + ObservedRole::Full | ObservedRole::Light if self.lucky_peers.len() < MIN_LUCKY => { + self.lucky_peers.insert(who.clone()); + }, + _ => {} } - self.inner.insert(who, PeerInfo::new(roles)); + self.inner.insert(who, PeerInfo::new(role)); } fn peer_disconnected(&mut self, who: &PeerId) { @@ -539,21 +542,28 @@ impl Peers { } fn authorities(&self) -> usize { - self.inner.iter().filter(|(_, info)| info.roles.is_authority()).count() + // Note that our sentry and our validator are neither authorities nor non-authorities. + self.inner.iter().filter(|(_, info)| matches!(info.roles, ObservedRole::Authority)).count() } fn non_authorities(&self) -> usize { - self.inner.iter().filter(|(_, info)| !info.roles.is_authority()).count() + // Note that our sentry and our validator are neither authorities nor non-authorities. + self.inner + .iter() + .filter(|(_, info)| matches!(info.roles, ObservedRole::Full | ObservedRole::Light)) + .count() } fn reshuffle(&mut self) { let mut lucky_peers: Vec<_> = self.inner .iter() - .filter_map(|(id, info)| if !info.roles.is_authority() { Some(id.clone()) } else { None }) + .filter_map(|(id, info)| + if matches!(info.roles, ObservedRole::Full | ObservedRole::Light) { Some(id.clone()) } else { None }) .collect(); let mut lucky_authorities: Vec<_> = self.inner .iter() - .filter_map(|(id, info)| if info.roles.is_authority() { Some(id.clone()) } else { None }) + .filter_map(|(id, info)| + if matches!(info.roles, ObservedRole::Authority) { Some(id.clone()) } else { None }) .collect(); let num_non_authorities = ((lucky_peers.len() as f32).sqrt() as usize) @@ -633,8 +643,11 @@ impl CatchUpConfig { fn request_allowed(&self, peer: &PeerInfo) -> bool { match self { CatchUpConfig::Disabled => false, - CatchUpConfig::Enabled { only_from_authorities, .. } => - !only_from_authorities || peer.roles.is_authority(), + CatchUpConfig::Enabled { only_from_authorities, .. } => match peer.roles { + ObservedRole::Authority | ObservedRole::OurSentry | + ObservedRole::OurGuardedAuthority => true, + _ => !only_from_authorities + } } } } @@ -1121,34 +1134,38 @@ impl Inner { return false; } - if peer.roles.is_authority() { - let authorities = self.peers.authorities(); + match peer.roles { + ObservedRole::OurGuardedAuthority | ObservedRole::OurSentry => true, + ObservedRole::Authority => { + let authorities = self.peers.authorities(); - // the target node is an authority, on the first round duration we start by - // sending the message to only `sqrt(authorities)` (if we're - // connected to at least `MIN_LUCKY`). - if round_elapsed < round_duration * PROPAGATION_ALL_AUTHORITIES - && authorities > MIN_LUCKY - { - self.peers.lucky_authorities.contains(who) - } else { - // otherwise we already went through the step above, so - // we won't filter the message and send it to all - // authorities for whom it is polite to do so - true - } - } else { - // the node is not an authority so we apply stricter filters - if round_elapsed >= round_duration * PROPAGATION_ALL { - // if we waited for 3 (or more) rounds - // then it is allowed to be sent to all peers. - true - } else if round_elapsed >= round_duration * PROPAGATION_SOME_NON_AUTHORITIES { - // otherwise we only send it to `sqrt(non-authorities)`. - self.peers.lucky_peers.contains(who) - } else { - false - } + // the target node is an authority, on the first round duration we start by + // sending the message to only `sqrt(authorities)` (if we're + // connected to at least `MIN_LUCKY`). + if round_elapsed < round_duration * PROPAGATION_ALL_AUTHORITIES + && authorities > MIN_LUCKY + { + self.peers.lucky_authorities.contains(who) + } else { + // otherwise we already went through the step above, so + // we won't filter the message and send it to all + // authorities for whom it is polite to do so + true + } + }, + ObservedRole::Full | ObservedRole::Light => { + // the node is not an authority so we apply stricter filters + if round_elapsed >= round_duration * PROPAGATION_ALL { + // if we waited for 3 (or more) rounds + // then it is allowed to be sent to all peers. + true + } else if round_elapsed >= round_duration * PROPAGATION_SOME_NON_AUTHORITIES { + // otherwise we only send it to `sqrt(non-authorities)`. + self.peers.lucky_peers.contains(who) + } else { + false + } + }, } } @@ -1170,38 +1187,42 @@ impl Inner { let round_duration = self.config.gossip_duration * ROUND_DURATION; let round_elapsed = self.round_start.elapsed(); - if peer.roles.is_authority() { - let authorities = self.peers.authorities(); - - // the target node is an authority, on the first round duration we start by - // sending the message to only `sqrt(authorities)` (if we're - // connected to at least `MIN_LUCKY`). - if round_elapsed < round_duration * PROPAGATION_ALL_AUTHORITIES - && authorities > MIN_LUCKY - { - self.peers.lucky_authorities.contains(who) - } else { - // otherwise we already went through the step above, so - // we won't filter the message and send it to all - // authorities for whom it is polite to do so - true - } - } else { - let non_authorities = self.peers.non_authorities(); - - // the target node is not an authority, on the first and second - // round duration we start by sending the message to only - // `sqrt(non_authorities)` (if we're connected to at least - // `MIN_LUCKY`). - if round_elapsed < round_duration * PROPAGATION_SOME_NON_AUTHORITIES - && non_authorities > MIN_LUCKY - { - self.peers.lucky_peers.contains(who) - } else { - // otherwise we already went through the step above, so - // we won't filter the message and send it to all - // non-authorities for whom it is polite to do so - true + match peer.roles { + ObservedRole::OurSentry | ObservedRole::OurGuardedAuthority => true, + ObservedRole::Authority => { + let authorities = self.peers.authorities(); + + // the target node is an authority, on the first round duration we start by + // sending the message to only `sqrt(authorities)` (if we're + // connected to at least `MIN_LUCKY`). + if round_elapsed < round_duration * PROPAGATION_ALL_AUTHORITIES + && authorities > MIN_LUCKY + { + self.peers.lucky_authorities.contains(who) + } else { + // otherwise we already went through the step above, so + // we won't filter the message and send it to all + // authorities for whom it is polite to do so + true + } + }, + ObservedRole::Full | ObservedRole::Light => { + let non_authorities = self.peers.non_authorities(); + + // the target node is not an authority, on the first and second + // round duration we start by sending the message to only + // `sqrt(non_authorities)` (if we're connected to at least + // `MIN_LUCKY`). + if round_elapsed < round_duration * PROPAGATION_SOME_NON_AUTHORITIES + && non_authorities > MIN_LUCKY + { + self.peers.lucky_peers.contains(who) + } else { + // otherwise we already went through the step above, so + // we won't filter the message and send it to all + // non-authorities for whom it is polite to do so + true + } } } } @@ -1233,7 +1254,7 @@ impl Metrics { pub(super) struct GossipValidator { inner: parking_lot::RwLock>, set_state: environment::SharedVoterSetState, - report_sender: mpsc::UnboundedSender, + report_sender: TracingUnboundedSender, metrics: Option, } @@ -1245,7 +1266,7 @@ impl GossipValidator { config: crate::Config, set_state: environment::SharedVoterSetState, prometheus_registry: Option<&Registry>, - ) -> (GossipValidator, mpsc::UnboundedReceiver) { + ) -> (GossipValidator, TracingUnboundedReceiver) { let metrics = match prometheus_registry.map(Metrics::register) { Some(Ok(metrics)) => Some(metrics), Some(Err(e)) => { @@ -1255,7 +1276,7 @@ impl GossipValidator { None => None, }; - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_grandpa_gossip_validator"); let val = GossipValidator { inner: parking_lot::RwLock::new(Inner::new(config)), set_state, @@ -1397,7 +1418,7 @@ impl GossipValidator { } impl sc_network_gossip::Validator for GossipValidator { - fn new_peer(&self, context: &mut dyn ValidatorContext, who: &PeerId, roles: Roles) { + fn new_peer(&self, context: &mut dyn ValidatorContext, who: &PeerId, roles: ObservedRole) { let packet = { let mut inner = self.inner.write(); inner.peers.new_peer(who.clone(), roles); @@ -1657,7 +1678,7 @@ mod tests { assert!(res.unwrap().is_none()); // connect & disconnect. - peers.new_peer(id.clone(), Roles::AUTHORITY); + peers.new_peer(id.clone(), ObservedRole::Authority); peers.peer_disconnected(&id); let res = peers.update_peer_state(&id, update.clone()); @@ -1693,7 +1714,7 @@ mod tests { let mut peers = Peers::default(); let id = PeerId::random(); - peers.new_peer(id.clone(), Roles::AUTHORITY); + peers.new_peer(id.clone(), ObservedRole::Authority); let mut check_update = move |update: NeighborPacket<_>| { let view = peers.update_peer_state(&id, update.clone()).unwrap().unwrap(); @@ -1713,7 +1734,7 @@ mod tests { let mut peers = Peers::default(); let id = PeerId::random(); - peers.new_peer(id.clone(), Roles::AUTHORITY); + peers.new_peer(id.clone(), ObservedRole::Authority); peers.update_peer_state(&id, NeighborPacket { round: Round(10), @@ -1914,7 +1935,7 @@ mod tests { // add the peer making the request to the validator, // otherwise it is discarded let mut inner = val.inner.write(); - inner.peers.new_peer(peer.clone(), Roles::AUTHORITY); + inner.peers.new_peer(peer.clone(), ObservedRole::Authority); let res = inner.handle_catch_up_request( &peer, @@ -1965,7 +1986,7 @@ mod tests { // add the peer making the request to the validator, // otherwise it is discarded let peer = PeerId::random(); - val.inner.write().peers.new_peer(peer.clone(), Roles::AUTHORITY); + val.inner.write().peers.new_peer(peer.clone(), ObservedRole::Authority); let send_request = |set_id, round| { let mut inner = val.inner.write(); @@ -2045,7 +2066,7 @@ mod tests { // add the peer making the request to the validator, // otherwise it is discarded. let peer = PeerId::random(); - val.inner.write().peers.new_peer(peer.clone(), Roles::AUTHORITY); + val.inner.write().peers.new_peer(peer.clone(), ObservedRole::Authority); let import_neighbor_message = |set_id, round| { let (_, _, catch_up_request, _) = val.inner.write().import_neighbor_message( @@ -2119,7 +2140,7 @@ mod tests { // add the peer making the request to the validator, // otherwise it is discarded. let peer = PeerId::random(); - val.inner.write().peers.new_peer(peer.clone(), Roles::AUTHORITY); + val.inner.write().peers.new_peer(peer.clone(), ObservedRole::Authority); // importing a neighbor message from a peer in the same set in a later // round should lead to a catch up request but since they're disabled @@ -2155,8 +2176,8 @@ mod tests { let peer_authority = PeerId::random(); let peer_full = PeerId::random(); - val.inner.write().peers.new_peer(peer_authority.clone(), Roles::AUTHORITY); - val.inner.write().peers.new_peer(peer_full.clone(), Roles::FULL); + val.inner.write().peers.new_peer(peer_authority.clone(), ObservedRole::Authority); + val.inner.write().peers.new_peer(peer_full.clone(), ObservedRole::Full); let import_neighbor_message = |peer| { let (_, _, catch_up_request, _) = val.inner.write().import_neighbor_message( @@ -2213,7 +2234,7 @@ mod tests { // add the peer making the requests to the validator, otherwise it is // discarded. let peer_full = PeerId::random(); - val.inner.write().peers.new_peer(peer_full.clone(), Roles::FULL); + val.inner.write().peers.new_peer(peer_full.clone(), ObservedRole::Full); let (_, _, catch_up_request, _) = val.inner.write().import_neighbor_message( &peer_full, @@ -2290,8 +2311,8 @@ mod tests { full_nodes.resize_with(30, || PeerId::random()); for i in 0..30 { - val.inner.write().peers.new_peer(authorities[i].clone(), Roles::AUTHORITY); - val.inner.write().peers.new_peer(full_nodes[i].clone(), Roles::FULL); + val.inner.write().peers.new_peer(authorities[i].clone(), ObservedRole::Authority); + val.inner.write().peers.new_peer(full_nodes[i].clone(), ObservedRole::Full); } let test = |num_round, peers| { @@ -2363,7 +2384,7 @@ mod tests { let mut authorities = Vec::new(); for _ in 0..5 { let peer_id = PeerId::random(); - val.inner.write().peers.new_peer(peer_id.clone(), Roles::AUTHORITY); + val.inner.write().peers.new_peer(peer_id.clone(), ObservedRole::Authority); authorities.push(peer_id); } @@ -2403,7 +2424,7 @@ mod tests { let mut authorities = Vec::new(); for _ in 0..100 { let peer_id = PeerId::random(); - val.inner.write().peers.new_peer(peer_id.clone(), Roles::AUTHORITY); + val.inner.write().peers.new_peer(peer_id.clone(), ObservedRole::Authority); authorities.push(peer_id); } @@ -2454,7 +2475,7 @@ mod tests { val.inner .write() .peers - .new_peer(peer1.clone(), Roles::AUTHORITY); + .new_peer(peer1.clone(), ObservedRole::Authority); val.inner .write() @@ -2474,7 +2495,7 @@ mod tests { val.inner .write() .peers - .new_peer(peer2.clone(), Roles::AUTHORITY); + .new_peer(peer2.clone(), ObservedRole::Authority); // create a commit for round 1 of set id 1 // targeting a block at height 2 diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 52bfdbc81881762077c6d0c61e88ea483f8091e8..7daa121513416087b8c25d4664acc8e25acaffc4 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -58,6 +58,7 @@ use gossip::{ use sp_finality_grandpa::{ AuthorityPair, AuthorityId, AuthoritySignature, SetId as SetIdNumber, RoundNumber, }; +use sp_utils::mpsc::TracingUnboundedReceiver; pub mod gossip; mod periodic; @@ -165,7 +166,7 @@ pub(crate) struct NetworkBridge> { // thus one has to wrap gossip_validator_report_stream with an `Arc` `Mutex`. Given that it is // just an `UnboundedReceiver`, one could also switch to a multi-producer-*multi*-consumer // channel implementation. - gossip_validator_report_stream: Arc>>, + gossip_validator_report_stream: Arc>>, } impl> Unpin for NetworkBridge {} diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index f2e79e8f14486e5bb0951acfd701e44aaec5a28c..f894624bdf79e6e000f7a113c804517633301cb9 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -17,9 +17,10 @@ //! Periodic rebroadcast of neighbor packets. use futures_timer::Delay; -use futures::{channel::mpsc, future::{FutureExt as _}, prelude::*, ready, stream::Stream}; +use futures::{future::{FutureExt as _}, prelude::*, ready, stream::Stream}; use log::debug; use std::{pin::Pin, task::{Context, Poll}, time::Duration}; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sc_network::PeerId; use sp_runtime::traits::{NumberFor, Block as BlockT}; @@ -31,7 +32,7 @@ const REBROADCAST_AFTER: Duration = Duration::from_secs(2 * 60); /// A sender used to send neighbor packets to a background job. #[derive(Clone)] pub(super) struct NeighborPacketSender( - mpsc::UnboundedSender<(Vec, NeighborPacket>)> + TracingUnboundedSender<(Vec, NeighborPacket>)> ); impl NeighborPacketSender { @@ -54,14 +55,15 @@ impl NeighborPacketSender { pub(super) struct NeighborPacketWorker { last: Option<(Vec, NeighborPacket>)>, delay: Delay, - rx: mpsc::UnboundedReceiver<(Vec, NeighborPacket>)>, + rx: TracingUnboundedReceiver<(Vec, NeighborPacket>)>, } impl Unpin for NeighborPacketWorker {} impl NeighborPacketWorker { pub(super) fn new() -> (Self, NeighborPacketSender){ - let (tx, rx) = mpsc::unbounded::<(Vec, NeighborPacket>)>(); + let (tx, rx) = tracing_unbounded::<(Vec, NeighborPacket>)> + ("mpsc_grandpa_neighbor_packet_worker"); let delay = Delay::new(REBROADCAST_AFTER); (NeighborPacketWorker { diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index 7c9170a48e5d1199a264e2697bad2372f15b7edc..ea995eff6390dd789632b0635fdd21b598561478 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -16,9 +16,9 @@ //! Tests for the communication portion of the GRANDPA crate. -use futures::channel::mpsc; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use futures::prelude::*; -use sc_network::{Event as NetworkEvent, PeerId, config::Roles}; +use sc_network::{Event as NetworkEvent, ObservedRole, PeerId}; use sc_network_test::{Block, Hash}; use sc_network_gossip::Validator; use std::sync::Arc; @@ -33,7 +33,7 @@ use super::{AuthorityId, VoterSet, Round, SetId}; #[derive(Debug)] pub(crate) enum Event { - EventStream(mpsc::UnboundedSender), + EventStream(TracingUnboundedSender), WriteNotification(sc_network::PeerId, Vec), Report(sc_network::PeerId, sc_network::ReputationChange), Announce(Hash), @@ -41,12 +41,12 @@ pub(crate) enum Event { #[derive(Clone)] pub(crate) struct TestNetwork { - sender: mpsc::UnboundedSender, + sender: TracingUnboundedSender, } impl sc_network_gossip::Network for TestNetwork { fn event_stream(&self) -> Pin + Send>> { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("test"); let _ = self.sender.unbounded_send(Event::EventStream(tx)); Box::pin(rx) } @@ -97,7 +97,7 @@ impl sc_network_gossip::ValidatorContext for TestNetwork { pub(crate) struct Tester { pub(crate) net_handle: super::NetworkBridge, gossip_validator: Arc>, - pub(crate) events: mpsc::UnboundedReceiver, + pub(crate) events: TracingUnboundedReceiver, } impl Tester { @@ -161,7 +161,7 @@ pub(crate) fn make_test_network() -> ( impl Future, TestNetwork, ) { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("test"); let net = TestNetwork { sender: tx }; #[derive(Clone)] @@ -256,7 +256,7 @@ fn good_commit_leads_to_relay() { let test = make_test_network().0 .then(move |tester| { // register a peer. - tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); + tester.gossip_validator.new_peer(&mut NoopContext, &id, ObservedRole::Full); future::ready((tester, id)) }) .then(move |(tester, id)| { @@ -284,7 +284,7 @@ fn good_commit_leads_to_relay() { let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: sender_id.clone(), engine_id: GRANDPA_ENGINE_ID, - roles: Roles::FULL, + role: ObservedRole::Full, }); let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { @@ -297,7 +297,7 @@ fn good_commit_leads_to_relay() { let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: receiver_id.clone(), engine_id: GRANDPA_ENGINE_ID, - roles: Roles::FULL, + role: ObservedRole::Full, }); // Announce its local set has being on the current set id through a neighbor @@ -404,7 +404,7 @@ fn bad_commit_leads_to_report() { let test = make_test_network().0 .map(move |tester| { // register a peer. - tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); + tester.gossip_validator.new_peer(&mut NoopContext, &id, ObservedRole::Full); (tester, id) }) .then(move |(tester, id)| { @@ -431,7 +431,7 @@ fn bad_commit_leads_to_report() { let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: sender_id.clone(), engine_id: GRANDPA_ENGINE_ID, - roles: Roles::FULL, + role: ObservedRole::Full, }); let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { remote: sender_id.clone(), @@ -482,7 +482,7 @@ fn peer_with_higher_view_leads_to_catch_up_request() { let test = tester .map(move |tester| { // register a peer with authority role. - tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::AUTHORITY); + tester.gossip_validator.new_peer(&mut NoopContext, &id, ObservedRole::Authority); (tester, id) }) .then(move |(tester, id)| { diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index a0f37f20cb390926cb6c43662e1370772863313b..d3bbc1adb3cb951b2ce18df858e50a5b24b4731a 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -20,7 +20,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use log::{debug, warn, info}; +use log::{debug, warn}; use parity_scale_codec::{Decode, Encode}; use futures::prelude::*; use futures_timer::Delay; @@ -636,6 +636,7 @@ where self.client.clone(), incoming, "round", + None, ).map_err(Into::into)); // schedule network message cleanup when sink drops. @@ -910,6 +911,7 @@ where hash, number, (round, commit).into(), + false, ) } @@ -969,6 +971,7 @@ pub(crate) fn finalize_block( hash: Block::Hash, number: NumberFor, justification_or_commit: JustificationOrCommit, + initial_sync: bool, ) -> Result<(), CommandOrError>> where Block: BlockT, BE: Backend, @@ -1010,6 +1013,7 @@ pub(crate) fn finalize_block( hash, number, &is_descendent_of::(&*client, None), + initial_sync, ).map_err(|e| Error::Safety(e.to_string()))?; // check if this is this is the first finalization of some consensus changes @@ -1090,9 +1094,15 @@ pub(crate) fn finalize_block( let (new_id, set_ref) = authority_set.current(); if set_ref.len() > 16 { - info!("Applying GRANDPA set change to new set with {} authorities", set_ref.len()); + afg_log!(initial_sync, + "👴 Applying GRANDPA set change to new set with {} authorities", + set_ref.len(), + ); } else { - info!("Applying GRANDPA set change to new set {:?}", set_ref); + afg_log!(initial_sync, + "👴 Applying GRANDPA set change to new set {:?}", + set_ref, + ); } telemetry!(CONSENSUS_INFO; "afg.generating_new_authority_set"; diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index ef5cc739de36580b80c7c610c58cc14735fba0b2..c1e32dfa6cc9e3847c18030f6f88c666d182a611 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -16,18 +16,18 @@ use std::{sync::Arc, collections::HashMap}; -use log::{debug, trace, info}; +use log::{debug, trace}; use parity_scale_codec::Encode; -use futures::channel::mpsc; use parking_lot::RwLockWriteGuard; use sp_blockchain::{BlockStatus, well_known_cache_keys}; use sc_client_api::{backend::Backend, utils::is_descendent_of}; +use sp_utils::mpsc::TracingUnboundedSender; use sp_api::{TransactionFor}; use sp_consensus::{ BlockImport, Error as ConsensusError, - BlockCheckParams, BlockImportParams, ImportResult, JustificationImport, + BlockCheckParams, BlockImportParams, BlockOrigin, ImportResult, JustificationImport, SelectChain, }; use sp_finality_grandpa::{ConsensusLog, ScheduledChange, SetId, GRANDPA_ENGINE_ID}; @@ -57,7 +57,7 @@ pub struct GrandpaBlockImport { inner: Arc, select_chain: SC, authority_set: SharedAuthoritySet>, - send_voter_commands: mpsc::UnboundedSender>>, + send_voter_commands: TracingUnboundedSender>>, consensus_changes: SharedConsensusChanges>, authority_set_hard_forks: HashMap>>, _phantom: PhantomData, @@ -128,7 +128,12 @@ impl JustificationImport number: NumberFor, justification: Justification, ) -> Result<(), Self::Error> { - GrandpaBlockImport::import_justification(self, hash, number, justification, false) + // this justification was requested by the sync service, therefore we + // are not sure if it should enact a change or not. it could have been a + // request made as part of initial sync but that means the justification + // wasn't part of the block and was requested asynchronously, probably + // makes sense to log in that case. + GrandpaBlockImport::import_justification(self, hash, number, justification, false, false) } } @@ -250,6 +255,7 @@ where &self, block: &mut BlockImportParams>, hash: Block::Hash, + initial_sync: bool, ) -> Result, ConsensusError> { // when we update the authorities, we need to hold the lock // until the block is written to prevent a race if we need to restore @@ -324,7 +330,9 @@ where } let applied_changes = { - let forced_change_set = guard.as_mut().apply_forced_changes(hash, number, &is_descendent_of) + let forced_change_set = guard + .as_mut() + .apply_forced_changes(hash, number, &is_descendent_of, initial_sync) .map_err(|e| ConsensusError::ClientImport(e.to_string())) .map_err(ConsensusError::from)?; @@ -419,7 +427,10 @@ impl BlockImport Err(e) => return Err(ConsensusError::ClientImport(e.to_string()).into()), } - let pending_changes = self.make_authorities_changes(&mut block, hash)?; + // on initial sync we will restrict logging under info to avoid spam. + let initial_sync = block.origin == BlockOrigin::NetworkInitialSync; + + let pending_changes = self.make_authorities_changes(&mut block, hash, initial_sync)?; // we don't want to finalize on `inner.import_block` let mut justification = block.justification.take(); @@ -492,7 +503,15 @@ impl BlockImport match justification { Some(justification) => { - self.import_justification(hash, number, justification, needs_justification).unwrap_or_else(|err| { + let import_res = self.import_justification( + hash, + number, + justification, + needs_justification, + initial_sync, + ); + + import_res.unwrap_or_else(|err| { if needs_justification || enacts_consensus_change { debug!(target: "afg", "Imported block #{} that enacts authority set change with \ invalid justification: {:?}, requesting justification from peers.", number, err); @@ -536,7 +555,7 @@ impl GrandpaBlockImport, select_chain: SC, authority_set: SharedAuthoritySet>, - send_voter_commands: mpsc::UnboundedSender>>, + send_voter_commands: TracingUnboundedSender>>, consensus_changes: SharedConsensusChanges>, authority_set_hard_forks: Vec<(SetId, PendingChange>)>, ) -> GrandpaBlockImport { @@ -604,6 +623,7 @@ where number: NumberFor, justification: Justification, enacts_change: bool, + initial_sync: bool, ) -> Result<(), ConsensusError> { let justification = GrandpaJustification::decode_and_verify_finalizes( &justification, @@ -625,12 +645,17 @@ where hash, number, justification.into(), + initial_sync, ); match result { Err(CommandOrError::VoterCommand(command)) => { - info!(target: "afg", "Imported justification for block #{} that triggers \ - command {}, signaling voter.", number, command); + afg_log!(initial_sync, + "👴 Imported justification for block #{} that triggers \ + command {}, signaling voter.", + number, + command, + ); // send the command to the voter let _ = self.send_voter_commands.unbounded_send(command); diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 967584fa9b2b3e8fb9463ed1d87e4305f04c6fdf..6fab89ac68acb1f03ed742df06724b62ffb2d54a 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -55,7 +55,6 @@ use futures::prelude::*; use futures::StreamExt; use log::{debug, info}; -use futures::channel::mpsc; use sc_client_api::{ backend::{AuxStore, Backend}, LockImportRun, BlockchainEvents, CallExecutor, @@ -63,12 +62,14 @@ use sc_client_api::{ }; use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata}; use parity_scale_codec::{Decode, Encode}; +use prometheus_endpoint::{PrometheusError, Registry}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT, DigestFor, Zero}; use sc_keystore::KeyStorePtr; use sp_inherents::InherentDataProviders; use sp_consensus::{SelectChain, BlockImport}; use sp_core::Pair; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sc_telemetry::{telemetry, CONSENSUS_INFO, CONSENSUS_DEBUG}; use serde_json; @@ -83,6 +84,23 @@ use std::time::Duration; use std::pin::Pin; use std::task::{Poll, Context}; +// utility logging macro that takes as first argument a conditional to +// decide whether to log under debug or info level (useful to restrict +// logging under initial sync). +macro_rules! afg_log { + ($condition:expr, $($msg: expr),+ $(,)?) => { + { + let log_level = if $condition { + log::Level::Debug + } else { + log::Level::Info + }; + + log::log!(target: "afg", log_level, $($msg),+); + } + }; +} + mod authorities; mod aux_schema; mod communication; @@ -104,7 +122,7 @@ pub use voting_rule::{ }; use aux_schema::PersistentData; -use environment::{Environment, VoterSetState, Metrics}; +use environment::{Environment, VoterSetState}; use import::GrandpaBlockImport; use until_imported::UntilGlobalMessageBlocksImported; use communication::{NetworkBridge, Network as NetworkT}; @@ -378,7 +396,7 @@ pub struct LinkHalf { client: Arc, select_chain: SC, persistent_data: PersistentData, - voter_commands_rx: mpsc::UnboundedReceiver>>, + voter_commands_rx: TracingUnboundedReceiver>>, } /// Provider for the Grandpa authority set configured on the genesis block. @@ -475,7 +493,7 @@ where } )?; - let (voter_commands_tx, voter_commands_rx) = mpsc::unbounded(); + let (voter_commands_tx, voter_commands_rx) = tracing_unbounded("mpsc_grandpa_voter_command"); // create pending change objects with 0 delay and enacted on finality // (i.e. standard changes) for each authority set hard fork. @@ -519,6 +537,7 @@ fn global_communication( client: Arc, network: &NetworkBridge, keystore: &Option, + metrics: Option, ) -> ( impl Stream< Item = Result, CommandOrError>>, @@ -549,6 +568,7 @@ fn global_communication( client.clone(), global_in, "global", + metrics, ); let global_in = global_in.map_err(CommandOrError::from); @@ -595,7 +615,7 @@ pub struct GrandpaParams { /// The inherent data providers. pub inherent_data_providers: InherentDataProviders, /// If supplied, can be used to hook on telemetry connection established events. - pub telemetry_on_connect: Option>, + pub telemetry_on_connect: Option>, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, /// The prometheus metrics registry. @@ -696,13 +716,30 @@ pub fn run_grandpa_voter( Ok(future::select(voter_work, telemetry_task).map(drop)) } +struct Metrics { + environment: environment::Metrics, + until_imported: until_imported::Metrics, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Metrics { + environment: environment::Metrics::register(registry)?, + until_imported: until_imported::Metrics::register(registry)?, + }) + } +} + /// Future that powers the voter. #[must_use] struct VoterWork, SC, VR> { voter: Pin>>> + Send>>, env: Arc>, - voter_commands_rx: mpsc::UnboundedReceiver>>, + voter_commands_rx: TracingUnboundedReceiver>>, network: NetworkBridge, + + /// Prometheus metrics. + metrics: Option, } impl VoterWork @@ -722,9 +759,17 @@ where select_chain: SC, voting_rule: VR, persistent_data: PersistentData, - voter_commands_rx: mpsc::UnboundedReceiver>>, + voter_commands_rx: TracingUnboundedReceiver>>, prometheus_registry: Option, ) -> Self { + let metrics = match prometheus_registry.as_ref().map(Metrics::register) { + Some(Ok(metrics)) => Some(metrics), + Some(Err(e)) => { + debug!(target: "afg", "Failed to register metrics: {:?}", e); + None + } + None => None, + }; let voters = persistent_data.authority_set.current_authorities(); let env = Arc::new(Environment { @@ -738,10 +783,7 @@ where authority_set: persistent_data.authority_set.clone(), consensus_changes: persistent_data.consensus_changes.clone(), voter_set_state: persistent_data.set_state.clone(), - metrics: prometheus_registry.map(|registry| { - Metrics::register(®istry) - .expect("Other metrics would have failed to register before these; qed") - }), + metrics: metrics.as_ref().map(|m| m.environment.clone()), _phantom: PhantomData, }); @@ -752,6 +794,7 @@ where env, voter_commands_rx, network, + metrics, }; work.rebuild_voter(); work @@ -800,6 +843,7 @@ where self.env.client.clone(), &self.env.network, &self.env.config.keystore, + self.metrics.as_ref().map(|m| m.until_imported.clone()), ); let last_completed_round = completed_rounds.last(); diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 07eaf1db9f6e86bf48dae74db19ed85aacd16778..1e6c8ddf18820e4de9e56e26c15d53491c286889 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -18,7 +18,7 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use futures::{prelude::*, channel::mpsc}; +use futures::prelude::*; use finality_grandpa::{ BlockNumberOps, Error as GrandpaError, voter, voter_set::VoterSet @@ -27,8 +27,10 @@ use log::{debug, info, warn}; use sp_consensus::SelectChain; use sc_client_api::backend::Backend; +use sp_utils::mpsc::TracingUnboundedReceiver; use sp_runtime::traits::{NumberFor, Block as BlockT}; use sp_blockchain::HeaderMetadata; + use crate::{ global_communication, CommandOrError, CommunicationIn, Config, environment, LinkHalf, Error, aux_schema::PersistentData, VoterCommand, VoterSetState, @@ -122,6 +124,7 @@ fn grandpa_observer( finalized_hash, finalized_number, (round, commit).into(), + false, ) { Ok(_) => {}, Err(e) => return future::err(e), @@ -206,7 +209,7 @@ struct ObserverWork> { network: NetworkBridge, persistent_data: PersistentData, keystore: Option, - voter_commands_rx: mpsc::UnboundedReceiver>>, + voter_commands_rx: TracingUnboundedReceiver>>, _phantom: PhantomData, } @@ -223,7 +226,7 @@ where network: NetworkBridge, persistent_data: PersistentData, keystore: Option, - voter_commands_rx: mpsc::UnboundedReceiver>>, + voter_commands_rx: TracingUnboundedReceiver>>, ) -> Self { let mut work = ObserverWork { @@ -255,6 +258,7 @@ where self.client.clone(), &self.network, &self.keystore, + None, ); let last_finalized_number = self.client.info().finalized_number; @@ -375,6 +379,7 @@ mod tests { use super::*; use assert_matches::assert_matches; + use sp_utils::mpsc::tracing_unbounded; use crate::{aux_schema, communication::tests::{Event, make_test_network}}; use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt}; use sc_network::PeerId; @@ -411,7 +416,7 @@ mod tests { || Ok(vec![]), ).unwrap(); - let (_tx, voter_command_rx) = mpsc::unbounded(); + let (_tx, voter_command_rx) = tracing_unbounded(""); let observer = ObserverWork::new( client, tester.net_handle.clone(), diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 50800f478476f02d36a346a519af9bb081d18dad..d7d1d1e48d3a0f0f051b1c4a35f4a044ea8ae2a7 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -22,7 +22,7 @@ use sc_network_test::{ Block, Hash, TestNetFactory, BlockImportAdapter, Peer, PeersClient, PassThroughVerifier, PeersFullClient, }; -use sc_network::config::{ProtocolConfig, Roles, BoxFinalityProofRequestBuilder}; +use sc_network::config::{ProtocolConfig, BoxFinalityProofRequestBuilder}; use parking_lot::Mutex; use futures_timer::Delay; use tokio::runtime::{Runtime, Handle}; @@ -30,21 +30,17 @@ use sp_keyring::Ed25519Keyring; use sc_client::LongestChain; use sc_client_api::backend::TransactionFor; use sp_blockchain::Result; -use sp_api::{ApiRef, ApiErrorExt, Core, RuntimeVersion, ApiExt, StorageProof, ProvideRuntimeApi}; +use sp_api::{ApiRef, StorageProof, ProvideRuntimeApi}; use substrate_test_runtime_client::runtime::BlockNumber; use sp_consensus::{ BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, ImportResult, BlockImport, import_queue::{BoxJustificationImport, BoxFinalityProofImport}, }; -use std::{ - collections::{HashMap, HashSet}, - result, - pin::Pin, -}; +use std::{collections::{HashMap, HashSet}, pin::Pin}; use parity_scale_codec::Decode; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, HashFor}; use sp_runtime::generic::{BlockId, DigestItem}; -use sp_core::{H256, NativeOrEncoded, ExecutionContext, crypto::Public}; +use sp_core::{H256, crypto::Public}; use sp_finality_grandpa::{GRANDPA_ENGINE_ID, AuthorityList, GrandpaApi}; use sp_state_machine::{InMemoryBackend, prove_read, read_proof_check}; @@ -78,9 +74,8 @@ impl GrandpaTestNet { peers: Vec::with_capacity(n_peers), test_config, }; - let config = Self::default_config(); for _ in 0..n_peers { - net.add_full_peer(&config); + net.add_full_peer(); } net } @@ -99,10 +94,8 @@ impl TestNetFactory for GrandpaTestNet { } fn default_config() -> ProtocolConfig { - // the authority role ensures gossip hits all nodes here. - let mut config = ProtocolConfig::default(); - config.roles = Roles::AUTHORITY; - config + // This is unused. + ProtocolConfig::default() } fn make_verifier( @@ -214,87 +207,13 @@ impl ProvideRuntimeApi for TestApi { } } -impl Core for RuntimeApi { - fn Core_version_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<()>, - _: Vec, - ) -> Result> { - unimplemented!("Not required for testing!") - } - - fn Core_execute_block_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option, - _: Vec, - ) -> Result> { - unimplemented!("Not required for testing!") - } +sp_api::mock_impl_runtime_apis! { + impl GrandpaApi for RuntimeApi { + type Error = sp_blockchain::Error; - fn Core_initialize_block_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<&::Header>, - _: Vec, - ) -> Result> { - unimplemented!("Not required for testing!") - } -} - -impl ApiErrorExt for RuntimeApi { - type Error = sp_blockchain::Error; -} - -impl ApiExt for RuntimeApi { - type StateBackend = < - substrate_test_runtime_client::Backend as sc_client_api::backend::Backend - >::State; - - fn map_api_result result::Result, R, E>( - &self, - _: F - ) -> result::Result { - unimplemented!("Not required for testing!") - } - - fn runtime_version_at(&self, _: &BlockId) -> Result { - unimplemented!("Not required for testing!") - } - - fn record_proof(&mut self) { - unimplemented!("Not required for testing!") - } - - fn extract_proof(&mut self) -> Option { - unimplemented!("Not required for testing!") - } - - fn into_storage_changes( - &self, - _: &Self::StateBackend, - _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, - _: ::Hash, - ) -> std::result::Result, String> - where Self: Sized - { - unimplemented!("Not required for testing!") - } -} - -impl GrandpaApi for RuntimeApi { - fn GrandpaApi_grandpa_authorities_runtime_api_impl( - &self, - _: &BlockId, - _: ExecutionContext, - _: Option<()>, - _: Vec, - ) -> Result> { - Ok(self.inner.genesis_authorities.clone()).map(NativeOrEncoded::Native) + fn grandpa_authorities(&self) -> AuthorityList { + self.inner.genesis_authorities.clone() + } } } @@ -1074,7 +993,7 @@ fn voter_persists_its_votes() { use std::iter::FromIterator; use std::sync::atomic::{AtomicUsize, Ordering}; use futures::future; - use futures::channel::mpsc; + use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; let _ = env_logger::try_init(); let mut runtime = Runtime::new().unwrap(); @@ -1099,7 +1018,7 @@ fn voter_persists_its_votes() { // channel between the voter and the main controller. // sending a message on the `voter_tx` restarts the voter. - let (voter_tx, voter_rx) = mpsc::unbounded::<()>(); + let (voter_tx, voter_rx) = tracing_unbounded::<()>(""); let mut keystore_paths = Vec::new(); @@ -1112,7 +1031,7 @@ fn voter_persists_its_votes() { struct ResettableVoter { voter: Pin + Send + Unpin>>, - voter_rx: mpsc::UnboundedReceiver<()>, + voter_rx: TracingUnboundedReceiver<()>, net: Arc>, client: PeersClient, keystore: KeyStorePtr, @@ -1381,7 +1300,7 @@ fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { let peers = &[Ed25519Keyring::Alice]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 1); - net.add_light_peer(&GrandpaTestNet::default_config()); + net.add_light_peer(); // import block#1 WITH consensus data change. Light client ignores justification // && instead fetches finality proof for block #1 @@ -1458,7 +1377,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ run_to_completion(&mut runtime, 11, net.clone(), peers_a); // request finalization by light client - net.lock().add_light_peer(&GrandpaTestNet::default_config()); + net.lock().add_light_peer(); net.lock().block_until_sync(); // check block, finalized on light client diff --git a/client/finality-grandpa/src/until_imported.rs b/client/finality-grandpa/src/until_imported.rs index c3a52fcf56fe197602ba844e61500686820f42cf..40da7707b6735262d115015422ef3b01c69eebff 100644 --- a/client/finality-grandpa/src/until_imported.rs +++ b/client/finality-grandpa/src/until_imported.rs @@ -29,63 +29,134 @@ use super::{ }; use log::{debug, warn}; -use sc_client_api::{BlockImportNotification, ImportNotifications}; +use sp_utils::mpsc::TracingUnboundedReceiver; use futures::prelude::*; use futures::stream::Fuse; use futures_timer::Delay; -use futures::channel::mpsc::UnboundedReceiver; use finality_grandpa::voter; use parking_lot::Mutex; +use prometheus_endpoint::{ + Gauge, U64, PrometheusError, register, Registry, +}; +use sc_client_api::{BlockImportNotification, ImportNotifications}; +use sp_finality_grandpa::AuthorityId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use std::collections::{HashMap, VecDeque}; use std::pin::Pin; -use std::sync::{atomic::{AtomicUsize, Ordering}, Arc}; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; -use sp_finality_grandpa::AuthorityId; const LOG_PENDING_INTERVAL: Duration = Duration::from_secs(15); -// something which will block until imported. +/// Something that needs to be withheld until specific blocks are available. +/// +/// For example a GRANDPA commit message which is not of any use without the corresponding block +/// that it commits on. pub(crate) trait BlockUntilImported: Sized { - // the type that is blocked on. + /// The type that is blocked on. type Blocked; - /// new incoming item. For all internal items, - /// check if they require to be waited for. - /// if so, call the `Wait` closure. - /// if they are ready, call the `Ready` closure. - fn schedule_wait( + /// Check if a new incoming item needs awaiting until a block(s) is imported. + fn needs_waiting>( input: Self::Blocked, status_check: &S, - wait: Wait, - ready: Ready, - ) -> Result<(), Error> where - S: BlockStatusT, - Wait: FnMut(Block::Hash, NumberFor, Self), - Ready: FnMut(Self::Blocked); + ) -> Result, Error>; /// called when the wait has completed. The canonical number is passed through /// for further checks. fn wait_completed(self, canon_number: NumberFor) -> Option; } +/// Describes whether a given [`BlockUntilImported`] (a) should be discarded, (b) is waiting for +/// specific blocks to be imported or (c) is ready to be used. +/// +/// A reason for discarding a [`BlockUntilImported`] would be if a referenced block is perceived +/// under a different number than specified in the message. +pub(crate) enum DiscardWaitOrReady { + Discard, + Wait(Vec<(Block::Hash, NumberFor, W)>), + Ready(R), +} + +/// Prometheus metrics for the `UntilImported` queue. +// +// At a given point in time there can be more than one `UntilImported` queue. One can not register a +// metric twice, thus queues need to share the same Prometheus metrics instead of instantiating +// their own ones. +// +// When a queue is dropped it might still contain messages. In order for those to not distort the +// Prometheus metrics, the `Metric` struct cleans up after itself within its `Drop` implementation +// by subtracting the local_waiting_messages (the amount of messages left in the queue about to +// be dropped) from the global_waiting_messages gauge. +pub(crate) struct Metrics { + global_waiting_messages: Gauge, + local_waiting_messages: u64, +} + +impl Metrics { + pub(crate) fn register(registry: &Registry) -> Result { + Ok(Self { + global_waiting_messages: register(Gauge::new( + "finality_grandpa_until_imported_waiting_messages_number", + "Number of finality grandpa messages waiting within the until imported queue.", + )?, registry)?, + local_waiting_messages: 0, + }) + } + + fn waiting_messages_inc(&mut self) { + self.local_waiting_messages += 1; + self.global_waiting_messages.inc(); + } + + fn waiting_messages_dec(&mut self) { + self.local_waiting_messages -= 1; + self.global_waiting_messages.dec(); + } +} + + +impl Clone for Metrics { + fn clone(&self) -> Self { + Metrics { + global_waiting_messages: self.global_waiting_messages.clone(), + // When cloned, reset local_waiting_messages, so the global counter is not reduced a + // second time for the same messages on `drop` of the clone. + local_waiting_messages: 0, + } + } +} + +impl Drop for Metrics { + fn drop(&mut self) { + // Reduce the global counter by the amount of messages that were still left in the dropped + // queue. + self.global_waiting_messages.sub(self.local_waiting_messages) + } +} + /// Buffering imported messages until blocks with given hashes are imported. #[pin_project::pin_project] pub(crate) struct UntilImported> { - import_notifications: Fuse>>, + import_notifications: Fuse>>, block_sync_requester: BlockSyncRequester, status_check: BlockStatus, #[pin] inner: Fuse, ready: VecDeque, + /// Interval at which to check status of each awaited block. check_pending: Pin> + Send>>, /// Mapping block hashes to their block number, the point in time it was /// first encountered (Instant) and a list of GRANDPA messages referencing /// the block hash. pending: HashMap, Instant, Vec)>, + + /// Queue identifier for differentiation in logs. identifier: &'static str, + /// Prometheus metrics. + metrics: Option, } impl UntilImported where @@ -102,6 +173,7 @@ impl UntilImported, ) -> Self { // how often to check if pending messages that are waiting for blocks to be // imported can be checked. @@ -125,6 +197,7 @@ impl UntilImported Stream for UntilImported { // new input: schedule wait of any parts which require // blocks to be known. - let ready = &mut this.ready; - let pending = &mut this.pending; - M::schedule_wait( - input, - this.status_check, - |target_hash, target_number, wait| pending - .entry(target_hash) - .or_insert_with(|| (target_number, Instant::now(), Vec::new())) - .2 - .push(wait), - |ready_item| ready.push_back(ready_item), - )?; + match M::needs_waiting(input, this.status_check)? { + DiscardWaitOrReady::Discard => {}, + DiscardWaitOrReady::Wait(items) => { + for (target_hash, target_number, wait) in items { + this.pending + .entry(target_hash) + .or_insert_with(|| (target_number, Instant::now(), Vec::new())) + .2 + .push(wait) + } + }, + DiscardWaitOrReady::Ready(item) => this.ready.push_back(item), + } + + if let Some(metrics) = &mut this.metrics { + metrics.waiting_messages_inc(); + } } Poll::Pending => break, } @@ -231,6 +309,9 @@ impl Stream for UntilImported(hash: H, id: AuthorityId) impl BlockUntilImported for SignedMessage { type Blocked = Self; - fn schedule_wait( + fn needs_waiting>( msg: Self::Blocked, status_check: &BlockStatus, - mut wait: Wait, - mut ready: Ready, - ) -> Result<(), Error> where - BlockStatus: BlockStatusT, - Wait: FnMut(Block::Hash, NumberFor, Self), - Ready: FnMut(Self::Blocked), - { + ) -> Result, Error> { let (&target_hash, target_number) = msg.target(); if let Some(number) = status_check.block_number(target_hash)? { if number != target_number { warn_authority_wrong_target(target_hash, msg.id); + return Ok(DiscardWaitOrReady::Discard); } else { - ready(msg); + return Ok(DiscardWaitOrReady::Ready(msg)); } - } else { - wait(target_hash, target_number, msg) } - Ok(()) + return Ok(DiscardWaitOrReady::Wait(vec![(target_hash, target_number, msg)])) } fn wait_completed(self, canon_number: NumberFor) -> Option { @@ -307,8 +381,12 @@ pub(crate) type UntilVoteTargetImported { - inner: Arc<(AtomicUsize, Mutex>>)>, + inner: Arc>>>, target_number: NumberFor, } @@ -317,16 +395,10 @@ impl Unpin for BlockGlobalMessage {} impl BlockUntilImported for BlockGlobalMessage { type Blocked = CommunicationIn; - fn schedule_wait( + fn needs_waiting>( input: Self::Blocked, status_check: &BlockStatus, - mut wait: Wait, - mut ready: Ready, - ) -> Result<(), Error> where - BlockStatus: BlockStatusT, - Wait: FnMut(Block::Hash, NumberFor, Self), - Ready: FnMut(Self::Blocked), - { + ) -> Result, Error> { use std::collections::hash_map::Entry; enum KnownOrUnknown { @@ -344,7 +416,6 @@ impl BlockUntilImported for BlockGlobalMessage { } let mut checked_hashes: HashMap<_, KnownOrUnknown>> = HashMap::new(); - let mut unknown_count = 0; { // returns false when should early exit. @@ -359,7 +430,6 @@ impl BlockUntilImported for BlockGlobalMessage { } else { entry.insert(KnownOrUnknown::Unknown(perceived_number)); - unknown_count += 1; perceived_number } } @@ -384,7 +454,7 @@ impl BlockUntilImported for BlockGlobalMessage { for (target_number, target_hash) in precommit_targets { if !query_known(target_hash, target_number)? { - return Ok(()) + return Ok(DiscardWaitOrReady::Discard); } } }, @@ -402,66 +472,52 @@ impl BlockUntilImported for BlockGlobalMessage { for (target_number, target_hash) in targets { if !query_known(target_hash, target_number)? { - return Ok(()) + return Ok(DiscardWaitOrReady::Discard); } } }, }; } - // none of the hashes in the global message were unknown. - // we can just return the message directly. - if unknown_count == 0 { - ready(input); - return Ok(()) + let unknown_hashes = checked_hashes.into_iter().filter_map(|(hash, num)| match num { + KnownOrUnknown::Unknown(number) => Some((hash, number)), + KnownOrUnknown::Known(_) => None, + }).collect::>(); + + if unknown_hashes.is_empty() { + // none of the hashes in the global message were unknown. + // we can just return the message directly. + return Ok(DiscardWaitOrReady::Ready(input)); } - let locked_global = Arc::new((AtomicUsize::new(unknown_count), Mutex::new(Some(input)))); + let locked_global = Arc::new(Mutex::new(Some(input))); + + let items_to_await = unknown_hashes.into_iter().map(|(hash, target_number)| { + (hash, target_number, BlockGlobalMessage { inner: locked_global.clone(), target_number }) + }).collect(); // schedule waits for all unknown messages. // when the last one of these has `wait_completed` called on it, // the global message will be returned. - // - // in the future, we may want to issue sync requests to the network - // if this is taking a long time. - for (hash, is_known) in checked_hashes { - if let KnownOrUnknown::Unknown(target_number) = is_known { - wait(hash, target_number, BlockGlobalMessage { - inner: locked_global.clone(), - target_number, - }) - } - } - - Ok(()) + Ok(DiscardWaitOrReady::Wait(items_to_await)) } fn wait_completed(self, canon_number: NumberFor) -> Option { if self.target_number != canon_number { - // if we return without deducting the counter, then none of the other - // handles can return the commit message. + // Delete the inner message so it won't ever be forwarded. Future calls to + // `wait_completed` on the same `inner` will ignore it. + *self.inner.lock() = None; return None; } - let mut last_count = self.inner.0.load(Ordering::Acquire); - - // CAS loop to ensure that we always have a last reader. - loop { - if last_count == 1 { // we are the last one left. - return self.inner.1.lock().take(); - } - - let prev_value = self.inner.0.compare_and_swap( - last_count, - last_count - 1, - Ordering::SeqCst, - ); - - if prev_value == last_count { - return None; - } else { - last_count = prev_value; - } + match Arc::try_unwrap(self.inner) { + // This is the last reference and thus the last outstanding block to be awaited. `inner` + // is either `Some(_)` or `None`. The latter implies that a previous `wait_completed` + // call witnessed a block number mismatch (see above). + Ok(inner) => Mutex::into_inner(inner), + // There are still other strong references to this `Arc`, thus the message is blocked on + // other blocks to be imported. + Err(_) => None, } } } @@ -485,18 +541,18 @@ mod tests { use sc_client_api::BlockImportNotification; use futures::future::Either; use futures_timer::Delay; - use futures::channel::mpsc; + use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use finality_grandpa::Precommit; #[derive(Clone)] struct TestChainState { - sender: mpsc::UnboundedSender>, + sender: TracingUnboundedSender>, known_blocks: Arc>>, } impl TestChainState { fn new() -> (Self, ImportNotifications) { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("test"); let state = TestChainState { sender: tx, known_blocks: Arc::new(Mutex::new(HashMap::new())), @@ -593,7 +649,7 @@ mod tests { // enact all dependencies before importing the message enact_dependencies(&chain_state); - let (global_tx, global_rx) = futures::channel::mpsc::unbounded(); + let (global_tx, global_rx) = tracing_unbounded("test"); let until_imported = UntilGlobalMessageBlocksImported::new( import_notifications, @@ -601,6 +657,7 @@ mod tests { block_status, global_rx, "global", + None, ); global_tx.unbounded_send(msg).unwrap(); @@ -619,7 +676,7 @@ mod tests { let (chain_state, import_notifications) = TestChainState::new(); let block_status = chain_state.block_status(); - let (global_tx, global_rx) = futures::channel::mpsc::unbounded(); + let (global_tx, global_rx) = tracing_unbounded("test"); let until_imported = UntilGlobalMessageBlocksImported::new( import_notifications, @@ -627,6 +684,7 @@ mod tests { block_status, global_rx, "global", + None, ); global_tx.unbounded_send(msg).unwrap(); @@ -871,7 +929,7 @@ mod tests { let (chain_state, import_notifications) = TestChainState::new(); let block_status = chain_state.block_status(); - let (global_tx, global_rx) = futures::channel::mpsc::unbounded(); + let (global_tx, global_rx) = tracing_unbounded("test"); let block_sync_requester = TestBlockSyncRequester::default(); @@ -881,6 +939,7 @@ mod tests { block_status, global_rx, "global", + None, ); let h1 = make_header(5); @@ -941,4 +1000,88 @@ mod tests { futures::executor::block_on(test); } + + fn test_catch_up() -> Arc>>> { + let header = make_header(5); + + let unknown_catch_up = finality_grandpa::CatchUp { + round_number: 1, + precommits: vec![], + prevotes: vec![], + base_hash: header.hash(), + base_number: *header.number(), + }; + + let catch_up = voter::CommunicationIn::CatchUp( + unknown_catch_up.clone(), + voter::Callback::Blank, + ); + + Arc::new(Mutex::new(Some(catch_up))) + } + + #[test] + fn block_global_message_wait_completed_return_when_all_awaited() { + let msg_inner = test_catch_up(); + + let waiting_block_1 = BlockGlobalMessage:: { + inner: msg_inner.clone(), + target_number: 1, + }; + + let waiting_block_2 = BlockGlobalMessage:: { + inner: msg_inner, + target_number: 2, + }; + + // waiting_block_2 is still waiting for block 2, thus this should return `None`. + assert!(waiting_block_1.wait_completed(1).is_none()); + + // Message only depended on block 1 and 2. Both have been imported, thus this should yield + // the message. + assert!(waiting_block_2.wait_completed(2).is_some()); + } + + #[test] + fn block_global_message_wait_completed_return_none_on_block_number_missmatch() { + let msg_inner = test_catch_up(); + + let waiting_block_1 = BlockGlobalMessage:: { + inner: msg_inner.clone(), + target_number: 1, + }; + + let waiting_block_2 = BlockGlobalMessage:: { + inner: msg_inner, + target_number: 2, + }; + + // Calling wait_completed with wrong block number should yield None. + assert!(waiting_block_1.wait_completed(1234).is_none()); + + // All blocks, that the message depended on, have been imported. Still, given the above + // block number mismatch this should return None. + assert!(waiting_block_2.wait_completed(2).is_none()); + } + + #[test] + fn metrics_cleans_up_after_itself() { + let r = Registry::new(); + + let mut m1 = Metrics::register(&r).unwrap(); + let m2 = m1.clone(); + + // Add a new message to the 'queue' of m1. + m1.waiting_messages_inc(); + + // m1 and m2 are synced through the shared atomic. + assert_eq!(1, m2.global_waiting_messages.get()); + + // Drop 'queue' m1. + drop(m1); + + // Make sure m1 cleaned up after itself, removing all messages that were left in its queue + // when dropped from the global metric. + assert_eq!(0, m2.global_waiting_messages.get()); + } } diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml index 9f3a93fbae53af54eec7e83a6a304b75c8ea9734..b60886d6dea6cfbb7d4b9151f2a2498b81be4404 100644 --- a/client/informant/Cargo.toml +++ b/client/informant/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] ansi_term = "0.12.1" -futures = "0.3.1" +futures = "0.3.4" log = "0.4.8" parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } wasm-timer = "0.2" @@ -19,3 +19,6 @@ sc-network = { version = "0.8.0-alpha.5", path = "../network" } sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../service" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/informant/src/display.rs b/client/informant/src/display.rs index 53c9697868348cb9a73328d26a459601196cd240..42f498998362e6eb1a6d02f4af1fc98ce285030f 100644 --- a/client/informant/src/display.rs +++ b/client/informant/src/display.rs @@ -68,24 +68,24 @@ impl InformantDisplay { self.last_number = Some(best_number); let (status, target) = match (net_status.sync_state, net_status.best_seen_block) { - (SyncState::Idle, _) => ("Idle".into(), "".into()), - (SyncState::Downloading, None) => (format!("Syncing{}", speed), "".into()), - (SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed), format!(", target=#{}", n)), + (SyncState::Idle, _) => ("💤 Idle".into(), "".into()), + (SyncState::Downloading, None) => (format!("⚙️ Preparing{}", speed), "".into()), + (SyncState::Downloading, Some(n)) => (format!("⚙️ Syncing{}", speed), format!(", target=#{}", n)), }; if self.format == OutputFormat::Coloured { info!( target: "substrate", - "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), {} {}", Colour::White.bold().paint(&status), target, Colour::White.bold().paint(format!("{}", num_connected_peers)), - Colour::White.paint(format!("{}", best_number)), + Colour::White.bold().paint(format!("{}", best_number)), best_hash, - Colour::White.paint(format!("{}", finalized_number)), + Colour::White.bold().paint(format!("{}", finalized_number)), info.chain.finalized_hash, - TransferRateFormat(net_status.average_download_per_sec), - TransferRateFormat(net_status.average_upload_per_sec), + Colour::Green.paint(format!("⬇ {}", TransferRateFormat(net_status.average_download_per_sec))), + Colour::Red.paint(format!("⬆ {}", TransferRateFormat(net_status.average_upload_per_sec))), ); } else { info!( diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs index d104a64a2dbd121a00d27dde8de80f61fee0c12d..66d5ed41fb5d418a2bae0952dc263cdee6ff8c84 100644 --- a/client/informant/src/lib.rs +++ b/client/informant/src/lib.rs @@ -16,6 +16,7 @@ //! Console informant. Prints sync progress and block events. Runs on the calling thread. +use ansi_term::Colour; use sc_client_api::BlockchainEvents; use futures::prelude::*; use log::{info, warn, trace}; @@ -79,10 +80,10 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur match maybe_ancestor { Ok(ref ancestor) if ancestor.hash != *last_hash => info!( - "Reorg from #{},{} to #{},{}, common ancestor #{},{}", - last_num, last_hash, - n.header.number(), n.hash, - ancestor.number, ancestor.hash, + "♻️ Reorg on #{},{} to #{},{}, common ancestor #{},{}", + Colour::Red.bold().paint(format!("{}", last_num)), last_hash, + Colour::Green.bold().paint(format!("{}", n.header.number())), n.hash, + Colour::White.bold().paint(format!("{}", ancestor.number)), ancestor.hash, ), Ok(_) => {}, Err(e) => warn!("Error computing tree route: {}", e), @@ -94,7 +95,7 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur last_best = Some((n.header.number().clone(), n.hash.clone())); } - info!(target: "substrate", "Imported #{} ({})", n.header.number(), n.hash); + info!(target: "substrate", "✨ Imported #{} ({})", Colour::White.bold().paint(format!("{}", n.header.number())), n.hash); future::ready(()) }); diff --git a/client/keystore/Cargo.toml b/client/keystore/Cargo.toml index 5b709630ef8d2e043bf5a466adad114b607a9280..af9cdf81bfe9a472f12dd4284001b95934ce0bd7 100644 --- a/client/keystore/Cargo.toml +++ b/client/keystore/Cargo.toml @@ -22,3 +22,6 @@ parking_lot = "0.10.0" [dev-dependencies] tempfile = "3.1.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/keystore/src/lib.rs b/client/keystore/src/lib.rs index 087ddf326de0141cc011cf4b046779193b4ba2a1..f8bc93097113b300bc85373918021da742700221 100644 --- a/client/keystore/src/lib.rs +++ b/client/keystore/src/lib.rs @@ -17,10 +17,11 @@ //! Keystore (and session key management) for ed25519 based chains like Polkadot. #![warn(missing_docs)] - -use std::{collections::HashMap, path::PathBuf, fs::{self, File}, io::{self, Write}, sync::Arc}; +use std::{collections::{HashMap, HashSet}, path::PathBuf, fs::{self, File}, io::{self, Write}, sync::Arc}; use sp_core::{ - crypto::{KeyTypeId, Pair as PairT, Public, IsWrappedBy, Protected}, traits::BareCryptoStore, + crypto::{IsWrappedBy, CryptoTypePublicPair, KeyTypeId, Pair as PairT, Protected, Public}, + traits::{BareCryptoStore, BareCryptoStoreError as TraitError}, + Encode, }; use sp_application_crypto::{AppKey, AppPublic, AppPair, ed25519, sr25519}; use parking_lot::RwLock; @@ -44,6 +45,12 @@ pub enum Error { /// Invalid seed #[display(fmt="Invalid seed")] InvalidSeed, + /// Public key type is not supported + #[display(fmt="Key crypto type is not supported")] + KeyNotSupported(KeyTypeId), + /// Pair not found for public key and KeyTypeId + #[display(fmt="Pair not found for {} public key", "_0")] + PairNotFound(String), /// Keystore unavailable #[display(fmt="Keystore unavailable")] Unavailable, @@ -52,6 +59,21 @@ pub enum Error { /// Keystore Result pub type Result = std::result::Result; +impl From for TraitError { + fn from(error: Error) -> Self { + match error { + Error::KeyNotSupported(id) => TraitError::KeyNotSupported(id), + Error::PairNotFound(e) => TraitError::PairNotFound(e), + Error::InvalidSeed | Error::InvalidPhrase | Error::InvalidPassword => { + TraitError::ValidationError(error.to_string()) + }, + Error::Unavailable => TraitError::Unavailable, + Error::Io(e) => TraitError::Other(e.to_string()), + Error::Json(e) => TraitError::Other(e.to_string()), + } + } +} + impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { @@ -220,16 +242,35 @@ impl Store { self.key_pair_by_type::(IsWrappedBy::from_ref(public), Pair::ID).map(Into::into) } - /// Get public keys of all stored keys that match the given key type. - pub fn public_keys_by_type(&self, key_type: KeyTypeId) -> Result> { - let mut public_keys: Vec = self.additional.keys() - .filter_map(|(ty, public)| { - if *ty == key_type { - Some(TPublic::from_slice(public)) - } else { - None - } + /// Get public keys of all stored keys that match the key type. + /// + /// This will just use the type of the public key (a list of which to be returned) in order + /// to determine the key type. Unless you use a specialized application-type public key, then + /// this only give you keys registered under generic cryptography, and will not return keys + /// registered under the application type. + pub fn public_keys(&self) -> Result> { + self.raw_public_keys(Public::ID) + .map(|v| { + v.into_iter() + .map(|k| Public::from_slice(k.as_slice())) + .collect() }) + } + + /// Returns the file path for the given public key and key type. + fn key_file_path(&self, public: &[u8], key_type: KeyTypeId) -> Option { + let mut buf = self.path.as_ref()?.clone(); + let key_type = hex::encode(key_type.0); + let key = hex::encode(public); + buf.push(key_type + key.as_str()); + Some(buf) + } + + /// Returns a list of raw public keys filtered by `KeyTypeId` + fn raw_public_keys(&self, id: KeyTypeId) -> Result>> { + let mut public_keys: Vec> = self.additional.keys() + .into_iter() + .filter_map(|k| if k.0 == id { Some(k.1.clone()) } else { None }) .collect(); if let Some(path) = &self.path { @@ -241,8 +282,10 @@ impl Store { if let Some(name) = path.file_name().and_then(|n| n.to_str()) { match hex::decode(name) { Ok(ref hex) if hex.len() > 4 => { - if &hex[0..4] != &key_type.0 { continue } - let public = TPublic::from_slice(&hex[4..]); + if &hex[0..4] != &id.0 { + continue; + } + let public = hex[4..].to_vec(); public_keys.push(public); } _ => continue, @@ -253,71 +296,104 @@ impl Store { Ok(public_keys) } +} - /// Get public keys of all stored keys that match the key type. - /// - /// This will just use the type of the public key (a list of which to be returned) in order - /// to determine the key type. Unless you use a specialized application-type public key, then - /// this only give you keys registered under generic cryptography, and will not return keys - /// registered under the application type. - pub fn public_keys(&self) -> Result> { - self.public_keys_by_type::(Public::ID) - .map(|v| v.into_iter().map(Into::into).collect()) +impl BareCryptoStore for Store { + fn keys( + &self, + id: KeyTypeId + ) -> std::result::Result, TraitError> { + let raw_keys = self.raw_public_keys(id)?; + Ok(raw_keys.into_iter() + .fold(Vec::new(), |mut v, k| { + v.push(CryptoTypePublicPair(sr25519::CRYPTO_ID, k.clone())); + v.push(CryptoTypePublicPair(ed25519::CRYPTO_ID, k.clone())); + v + })) + } + + fn supported_keys( + &self, + id: KeyTypeId, + keys: Vec + ) -> std::result::Result, TraitError> { + let all_keys = self.keys(id)?.into_iter().collect::>(); + Ok(keys.into_iter() + .filter(|key| all_keys.contains(key)) + .collect::>()) } - /// Returns the file path for the given public key and key type. - fn key_file_path(&self, public: &[u8], key_type: KeyTypeId) -> Option { - let mut buf = self.path.as_ref()?.clone(); - let key_type = hex::encode(key_type.0); - let key = hex::encode(public); - buf.push(key_type + key.as_str()); - Some(buf) + fn sign_with( + &self, + id: KeyTypeId, + key: &CryptoTypePublicPair, + msg: &[u8], + ) -> std::result::Result, TraitError> { + match key.0 { + ed25519::CRYPTO_ID => { + let pub_key = ed25519::Public::from_slice(key.1.as_slice()); + let key_pair: ed25519::Pair = self + .key_pair_by_type::(&pub_key, id) + .map_err(|e| TraitError::from(e))?; + Ok(key_pair.sign(msg).encode()) + } + sr25519::CRYPTO_ID => { + let pub_key = sr25519::Public::from_slice(key.1.as_slice()); + let key_pair: sr25519::Pair = self + .key_pair_by_type::(&pub_key, id) + .map_err(|e| TraitError::from(e))?; + Ok(key_pair.sign(msg).encode()) + } + _ => Err(TraitError::KeyNotSupported(id)) + } } -} -impl BareCryptoStore for Store { fn sr25519_public_keys(&self, key_type: KeyTypeId) -> Vec { - self.public_keys_by_type::(key_type).unwrap_or_default() + self.raw_public_keys(key_type) + .map(|v| { + v.into_iter() + .map(|k| sr25519::Public::from_slice(k.as_slice())) + .collect() + }) + .unwrap_or_default() } fn sr25519_generate_new( &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> std::result::Result { + ) -> std::result::Result { let pair = match seed { Some(seed) => self.insert_ephemeral_from_seed_by_type::(seed, id), None => self.generate_by_type::(id), - }.map_err(|e| e.to_string())?; + }.map_err(|e| -> TraitError { e.into() })?; Ok(pair.public()) } - fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option { - self.key_pair_by_type::(pub_key, id).ok() - } - fn ed25519_public_keys(&self, key_type: KeyTypeId) -> Vec { - self.public_keys_by_type::(key_type).unwrap_or_default() + self.raw_public_keys(key_type) + .map(|v| { + v.into_iter() + .map(|k| ed25519::Public::from_slice(k.as_slice())) + .collect() + }) + .unwrap_or_default() } fn ed25519_generate_new( &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> std::result::Result { + ) -> std::result::Result { let pair = match seed { Some(seed) => self.insert_ephemeral_from_seed_by_type::(seed, id), None => self.generate_by_type::(id), - }.map_err(|e| e.to_string())?; + }.map_err(|e| -> TraitError { e.into() })?; Ok(pair.public()) } - fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option { - self.key_pair_by_type::(pub_key, id).ok() - } - fn insert_unknown(&mut self, key_type: KeyTypeId, suri: &str, public: &[u8]) -> std::result::Result<(), ()> { @@ -337,7 +413,7 @@ impl BareCryptoStore for Store { mod tests { use super::*; use tempfile::TempDir; - use sp_core::{testing::{SR25519}, crypto::{Ss58Codec}}; + use sp_core::{testing::SR25519, crypto::Ss58Codec}; #[test] fn basic_store() { @@ -451,7 +527,7 @@ mod tests { fs::write(file_name, "test").expect("Invalid file is written"); assert!( - store.read().public_keys_by_type::(SR25519).unwrap().is_empty(), + store.read().sr25519_public_keys(SR25519).is_empty(), ); } } diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index c0418125a449a98786cce5f85b5a73337757eb29..c2887ed71925facd68b91fc2f08b2424181680d1 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -11,14 +11,18 @@ documentation = "https://docs.rs/sc-network-gossip" [dependencies] -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.17.0", default-features = false, features = ["websocket"] } log = "0.4.8" lru = "0.4.3" sc-network = { version = "0.8.0-alpha.5", path = "../network" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } wasm-timer = "0.2" [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 0b3a58bba2746421828362e314dc8df6590e7cba..6a00b3d5a181ab812aea45007c42096535946a0f 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -19,10 +19,11 @@ use crate::state_machine::{ConsensusGossip, TopicNotification, PERIODIC_MAINTENA use sc_network::{Event, ReputationChange}; -use futures::{prelude::*, channel::mpsc}; +use futures::prelude::*; use libp2p::PeerId; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; use std::{borrow::Cow, pin::Pin, sync::Arc, task::{Context, Poll}}; +use sp_utils::mpsc::TracingUnboundedReceiver; /// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on /// top of it. @@ -86,7 +87,7 @@ impl GossipEngine { /// Get data of valid, incoming messages for a topic (but might have expired meanwhile). pub fn messages_for(&mut self, topic: B::Hash) - -> mpsc::UnboundedReceiver + -> TracingUnboundedReceiver { self.state_machine.messages_for(self.engine_id, topic) } @@ -137,11 +138,11 @@ impl Future for GossipEngine { loop { match this.network_event_stream.poll_next_unpin(cx) { Poll::Ready(Some(event)) => match event { - Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { + Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, role } => { if msg_engine_id != this.engine_id { continue; } - this.state_machine.new_peer(&mut *this.network, remote, roles); + this.state_machine.new_peer(&mut *this.network, remote, role); } Event::NotificationStreamClosed { remote, engine_id: msg_engine_id } => { if msg_engine_id != this.engine_id { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index e2d7ec45b57a4c57cdda8a566b3402f115df64dd..c846534488bf5a94760688089b8ecb1917cace62 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -21,12 +21,12 @@ use std::sync::Arc; use std::iter; use std::time; use log::trace; -use futures::channel::mpsc; use lru::LruCache; use libp2p::PeerId; use sp_runtime::traits::{Block as BlockT, Hash, HashFor}; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; use sp_runtime::ConsensusEngineId; -use sc_network::config::Roles; +use sc_network::ObservedRole; use wasm_timer::Instant; // FIXME: Add additional spam/DoS attack protection: https://github.com/paritytech/substrate/issues/1115 @@ -51,7 +51,7 @@ mod rep { struct PeerConsensus { known_messages: HashSet, - roles: Roles, + role: ObservedRole, } /// Topic stream message with sender. @@ -164,7 +164,7 @@ fn propagate<'a, B: BlockT, I>( /// Consensus network protocol handler. Manages statements and candidate requests. pub struct ConsensusGossip { peers: HashMap>, - live_message_sinks: HashMap<(ConsensusEngineId, B::Hash), Vec>>, + live_message_sinks: HashMap<(ConsensusEngineId, B::Hash), Vec>>, messages: Vec>, known_messages: LruCache, validators: HashMap>>, @@ -192,10 +192,10 @@ impl ConsensusGossip { validator: Arc> ) { self.register_validator_internal(engine_id, validator.clone()); - let peers: Vec<_> = self.peers.iter().map(|(id, peer)| (id.clone(), peer.roles)).collect(); - for (id, roles) in peers { + let peers: Vec<_> = self.peers.iter().map(|(id, peer)| (id.clone(), peer.role.clone())).collect(); + for (id, role) in peers { let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; - validator.new_peer(&mut context, &id, roles); + validator.new_peer(&mut context, &id, role); } } @@ -204,20 +204,20 @@ impl ConsensusGossip { } /// Handle new connected peer. - pub fn new_peer(&mut self, network: &mut dyn Network, who: PeerId, roles: Roles) { + pub fn new_peer(&mut self, network: &mut dyn Network, who: PeerId, role: ObservedRole) { // light nodes are not valid targets for consensus gossip messages - if !roles.is_full() { + if role.is_light() { return; } - trace!(target:"gossip", "Registering {:?} {}", roles, who); + trace!(target:"gossip", "Registering {:?} {}", role, who); self.peers.insert(who.clone(), PeerConsensus { known_messages: HashSet::new(), - roles, + role: role.clone(), }); for (engine_id, v) in self.validators.clone() { let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; - v.new_peer(&mut context, &who, roles); + v.new_peer(&mut context, &who, role.clone()); } } @@ -333,9 +333,9 @@ impl ConsensusGossip { /// Get data of valid, incoming messages for a topic (but might have expired meanwhile) pub fn messages_for(&mut self, engine_id: ConsensusEngineId, topic: B::Hash) - -> mpsc::UnboundedReceiver + -> TracingUnboundedReceiver { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_gossip_messages_for"); for entry in self.messages.iter_mut() .filter(|e| e.topic == topic && e.engine_id == engine_id) { @@ -696,7 +696,7 @@ mod tests { let mut network = TestNetwork; let peer_id = PeerId::random(); - consensus.new_peer(&mut network, peer_id.clone(), Roles::FULL); + consensus.new_peer(&mut network, peer_id.clone(), ObservedRole::Full); assert!(consensus.peers.contains_key(&peer_id)); consensus.peer_disconnected(&mut network, peer_id.clone()); diff --git a/client/network-gossip/src/validator.rs b/client/network-gossip/src/validator.rs index 74b5307ee9cdcdaf296a10dee8c4881a0495e105..6b330d7b618c0e702e52cc330ed2bd77b3ffdbdd 100644 --- a/client/network-gossip/src/validator.rs +++ b/client/network-gossip/src/validator.rs @@ -14,13 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_network::{config::Roles, PeerId}; +use sc_network::{ObservedRole, PeerId}; use sp_runtime::traits::Block as BlockT; /// Validates consensus messages. pub trait Validator: Send + Sync { /// New peer is connected. - fn new_peer(&self, _context: &mut dyn ValidatorContext, _who: &PeerId, _roles: Roles) { + fn new_peer(&self, _context: &mut dyn ValidatorContext, _who: &PeerId, _role: ObservedRole) { } /// New connection is dropped. diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index a5f71d87d57fbdef5ab625604f12965735147436..5b4813e80a5fd57228dead65954f477eb6140f47 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -16,17 +16,16 @@ prost-build = "0.6.1" [dependencies] bitflags = "1.2.0" bytes = "0.5.0" -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } derive_more = "0.99.2" either = "1.5.3" erased-serde = "0.3.9" fnv = "1.0.6" fork-tree = { version = "2.0.0-alpha.5", path = "../../utils/fork-tree" } -futures = "0.3.1" +futures = "0.3.4" futures_codec = "0.3.3" futures-timer = "3.0.1" wasm-timer = "0.2" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } linked-hash-map = "0.5.2" linked_hash_set = "0.1.3" log = "0.4.8" @@ -36,10 +35,10 @@ parking_lot = "0.10.0" prost = "0.6.1" rand = "0.7.2" hex = "0.4.0" -sc-block-builder = { version = "0.8.0-alpha.4", path = "../block-builder" } -sc-client = { version = "0.8.0-alpha.4", path = "../" } -sc-client-api = { version = "2.0.0-alpha.4", path = "../api" } -sc-peerset = { version = "2.0.0-alpha.4", path = "../peerset" } +sc-block-builder = { version = "0.8.0-alpha.5", path = "../block-builder" } +sc-client = { version = "0.8.0-alpha.5", path = "../" } +sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } +sc-peerset = { version = "2.0.0-alpha.5", path = "../peerset" } pin-project = "0.4.6" serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" @@ -47,6 +46,7 @@ slog = { version = "2.5.2", features = ["nested-values"] } slog_derive = "0.2.0" smallvec = "0.6.10" sp-arithmetic = { version = "2.0.0-alpha.5", path = "../../primitives/arithmetic" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/babe" } @@ -58,10 +58,16 @@ unsigned-varint = { version = "0.3.1", features = ["futures", "futures-codec"] } void = "1.0.2" zeroize = "1.0.0" +[dependencies.libp2p] +version = "0.17.0" +default-features = false +features = ["websocket", "kad", "mdns", "ping", "identify", "mplex", "yamux", "noise"] + [dev-dependencies] async-std = "1.5" assert_matches = "1.3" env_logger = "0.7.0" +libp2p = { version = "0.17.0", default-features = false, features = ["secio"] } quickcheck = "0.9.0" rand = "0.7.2" sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } @@ -73,3 +79,6 @@ tempfile = "3.1.0" [features] default = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index e7aca1975cd0d03f450e23528137db6511b1490e..203435134c6cd090bb84b9e6e8d08f384f0b79af 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -15,18 +15,19 @@ // along with Substrate. If not, see . use crate::{ - debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, DiscoveryNetBehaviour, - Event, protocol::event::DhtEvent, ExHashT, + config::Role, + debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, + Event, ObservedRole, DhtEvent, ExHashT, }; -use crate::protocol::{self, light_client_handler, CustomMessageOutcome, Protocol}; +use crate::protocol::{self, light_client_handler, message::Roles, CustomMessageOutcome, Protocol}; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; use libp2p::kad::record; use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters}; use log::debug; use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}}; -use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justification}; -use std::{iter, task::Context, task::Poll}; +use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId, Justification}; +use std::{borrow::Cow, iter, task::Context, task::Poll}; use void; /// General behaviour of the network. Combines all protocols together. @@ -44,9 +45,14 @@ pub struct Behaviour { block_requests: protocol::BlockRequests, /// Light client request handling. light_client_handler: protocol::LightClientHandler, + /// Queue of events to produce for the outside. #[behaviour(ignore)] events: Vec>, + + /// Role of our local node, as originally passed from the configuration. + #[behaviour(ignore)] + role: Role, } /// Event generated by `Behaviour`. @@ -63,6 +69,7 @@ impl Behaviour { /// Builds a new `Behaviour`. pub async fn new( substrate: Protocol, + role: Role, user_agent: String, local_public_key: PublicKey, known_addresses: Vec<(PeerId, Multiaddr)>, @@ -84,7 +91,8 @@ impl Behaviour { ).await, block_requests, light_client_handler, - events: Vec::new() + events: Vec::new(), + role, } } @@ -112,6 +120,32 @@ impl Behaviour { self.debug_info.node(peer_id) } + /// Registers a new notifications protocol. + /// + /// After that, you can call `write_notifications`. + /// + /// Please call `event_stream` before registering a protocol, otherwise you may miss events + /// about the protocol that you have registered. + /// + /// You are very strongly encouraged to call this method very early on. Any connection open + /// will retain the protocols that were registered then, and not any new one. + pub fn register_notifications_protocol( + &mut self, + engine_id: ConsensusEngineId, + protocol_name: impl Into>, + ) { + let list = self.substrate.register_notifications_protocol(engine_id, protocol_name); + for (remote, roles) in list { + let role = reported_roles_to_observed_role(&self.role, remote, roles); + let ev = Event::NotificationStreamOpened { + remote: remote.clone(), + engine_id, + role, + }; + self.events.push(BehaviourOut::Event(ev)); + } + } + /// Returns a shared reference to the user protocol. pub fn user_protocol(&self) -> &Protocol { &self.substrate @@ -133,12 +167,27 @@ impl Behaviour { } /// Issue a light client request. - #[allow(unused)] pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { self.light_client_handler.request(r) } } +fn reported_roles_to_observed_role(local_role: &Role, remote: &PeerId, roles: Roles) -> ObservedRole { + if roles.is_authority() { + match local_role { + Role::Authority { sentry_nodes } + if sentry_nodes.iter().any(|s| s.peer_id == *remote) => ObservedRole::OurSentry, + Role::Sentry { validators } + if validators.iter().any(|s| s.peer_id == *remote) => ObservedRole::OurGuardedAuthority, + _ => ObservedRole::Authority + } + } else if roles.is_full() { + ObservedRole::Full + } else { + ObservedRole::Light + } +} + impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: void::Void) { @@ -156,14 +205,16 @@ Behaviour { self.events.push(BehaviourOut::JustificationImport(origin, hash, nb, justification)), CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), - CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles } => + CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles } => { + let role = reported_roles_to_observed_role(&self.role, &remote, roles); for engine_id in protocols { self.events.push(BehaviourOut::Event(Event::NotificationStreamOpened { remote: remote.clone(), engine_id, - roles, + role: role.clone(), })); - }, + } + }, CustomMessageOutcome::NotificationStreamClosed { remote, protocols } => for engine_id in protocols { self.events.push(BehaviourOut::Event(Event::NotificationStreamClosed { @@ -175,6 +226,9 @@ Behaviour { let ev = Event::NotificationsReceived { remote, messages }; self.events.push(BehaviourOut::Event(ev)); }, + CustomMessageOutcome::PeerNewBest(peer_id, number) => { + self.light_client_handler.update_best_block(&peer_id, number); + } CustomMessageOutcome::None => {} } } diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 3f73d761ce879e7b693492a3a04e2f2087f4584b..9cd57a9d1d47bd9bfac6ec96c5a0ab1e8a8511d5 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -20,7 +20,7 @@ //! See the documentation of [`Params`]. pub use crate::chain::{Client, FinalityProofProvider}; -pub use crate::on_demand_layer::OnDemand; +pub use crate::on_demand_layer::{AlwaysBadChecker, OnDemand}; pub use crate::service::{TransactionPool, EmptyTransactionPool}; pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr}; @@ -31,23 +31,21 @@ pub use crate::protocol::ProtocolConfig; use crate::service::ExHashT; -use bitflags::bitflags; use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue}; use sp_runtime::traits::{Block as BlockT}; use libp2p::identity::{Keypair, ed25519}; use libp2p::wasm_ext; use libp2p::{PeerId, Multiaddr, multiaddr}; use core::{fmt, iter}; -use std::{future::Future, pin::Pin}; +use std::{convert::TryFrom, future::Future, pin::Pin, str::FromStr}; use std::{error::Error, fs, io::{self, Write}, net::Ipv4Addr, path::{Path, PathBuf}, sync::Arc}; use zeroize::Zeroize; use prometheus_endpoint::Registry; - /// Network initialization parameters. pub struct Params { - /// Assigned roles for our node (full, light, ...). - pub roles: Roles, + /// Assigned role for our node (full, light, ...). + pub role: Role, /// How to spawn background tasks. If you pass `None`, then a threads pool will be used by /// default. @@ -97,54 +95,48 @@ pub struct Params { pub metrics_registry: Option, } -bitflags! { - /// Bitmask of the roles that a node fulfills. - pub struct Roles: u8 { - /// No network. - const NONE = 0b00000000; - /// Full node, does not participate in consensus. - const FULL = 0b00000001; - /// Light client node. - const LIGHT = 0b00000010; - /// Act as an authority - const AUTHORITY = 0b00000100; +/// Role of the local node. +#[derive(Debug, Clone)] +pub enum Role { + /// Regular full node. + Full, + /// Regular light node. + Light, + /// Sentry node that guards an authority. Will be reported as "authority" on the wire protocol. + Sentry { + /// Address and identity of the validator nodes that we're guarding. + /// + /// The nodes will be granted some priviledged status. + validators: Vec, + }, + /// Actual authority. + Authority { + /// List of public addresses and identities of our sentry nodes. + sentry_nodes: Vec, } } -impl Roles { - /// Does this role represents a client that holds full chain data locally? - pub fn is_full(&self) -> bool { - self.intersects(Roles::FULL | Roles::AUTHORITY) - } - - /// Does this role represents a client that does not participates in the consensus? +impl Role { + /// True for `Role::Authority` pub fn is_authority(&self) -> bool { - *self == Roles::AUTHORITY + matches!(self, Role::Authority { .. }) } - /// Does this role represents a client that does not hold full chain data locally? - pub fn is_light(&self) -> bool { - !self.is_full() + /// True for `Role::Authority` and `Role::Sentry` since they're both + /// announced as having the authority role to the network. + pub fn is_network_authority(&self) -> bool { + matches!(self, Role::Authority { .. } | Role::Sentry { .. }) } } -impl fmt::Display for Roles { +impl fmt::Display for Role { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -impl codec::Encode for Roles { - fn encode_to(&self, dest: &mut T) { - dest.push_byte(self.bits()) - } -} - -impl codec::EncodeLike for Roles {} - -impl codec::Decode for Roles { - fn decode(input: &mut I) -> Result { - Self::from_bits(input.read_byte()?).ok_or_else(|| codec::Error::from("Invalid bytes")) + match self { + Role::Full => write!(f, "FULL"), + Role::Light => write!(f, "LIGHT"), + Role::Sentry { .. } => write!(f, "SENTRY"), + Role::Authority { .. } => write!(f, "AUTHORITY"), + } } } @@ -214,6 +206,67 @@ pub fn parse_addr(mut addr: Multiaddr)-> Result<(PeerId, Multiaddr), ParseErr> { Ok((who, addr)) } +/// Address of a node, including its identity. +/// +/// This struct represents a decoded version of a multiaddress that ends with `/p2p/`. +/// +/// # Example +/// +/// ``` +/// # use sc_network::{Multiaddr, PeerId, config::MultiaddrWithPeerId}; +/// let addr: MultiaddrWithPeerId = +/// "/ip4/198.51.100.19/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".parse().unwrap(); +/// assert_eq!(addr.peer_id.to_base58(), "QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV"); +/// assert_eq!(addr.multiaddr.to_string(), "/ip4/198.51.100.19/tcp/30333"); +/// ``` +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct MultiaddrWithPeerId { + /// Address of the node. + pub multiaddr: Multiaddr, + /// Its identity. + pub peer_id: PeerId, +} + +impl MultiaddrWithPeerId { + /// Concatenates the multiaddress and peer ID into one multiaddress containing both. + pub fn concat(&self) -> Multiaddr { + let proto = multiaddr::Protocol::P2p(From::from(self.peer_id.clone())); + self.multiaddr.clone().with(proto) + } +} + +impl fmt::Display for MultiaddrWithPeerId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.concat(), f) + } +} + +impl FromStr for MultiaddrWithPeerId { + type Err = ParseErr; + + fn from_str(s: &str) -> Result { + let (peer_id, multiaddr) = parse_str_addr(s)?; + Ok(MultiaddrWithPeerId { + peer_id, + multiaddr, + }) + } +} + +impl From for String { + fn from(ma: MultiaddrWithPeerId) -> String { + format!("{}", ma) + } +} + +impl TryFrom for MultiaddrWithPeerId { + type Error = ParseErr; + fn try_from(string: String) -> Result { + string.parse() + } +} + /// Error that can be generated by `parse_str_addr`. #[derive(Debug)] pub enum ParseErr { @@ -254,16 +307,14 @@ impl From for ParseErr { /// Network service configuration. #[derive(Clone, Debug)] pub struct NetworkConfiguration { - /// Directory path to store general network configuration. None means nothing will be saved. - pub config_path: Option, /// Directory path to store network-specific configuration. None means nothing will be saved. - pub net_config_path: Option, + pub net_config_path: PathBuf, /// Multiaddresses to listen for incoming connections. pub listen_addresses: Vec, /// Multiaddresses to advertise. Detected automatically if empty. pub public_addresses: Vec, /// List of initial node addresses - pub boot_nodes: Vec, + pub boot_nodes: Vec, /// The node key configuration, which determines the node's network identity keypair. pub node_key: NodeKeyConfig, /// Maximum allowed number of incoming connections. @@ -271,11 +322,9 @@ pub struct NetworkConfiguration { /// Number of outgoing connections we're trying to maintain. pub out_peers: u32, /// List of reserved node addresses. - pub reserved_nodes: Vec, + pub reserved_nodes: Vec, /// The non-reserved peer mode. pub non_reserved_mode: NonReservedPeerMode, - /// List of sentry node public addresses. - pub sentry_nodes: Vec, /// Client identifier. Sent over the wire for debugging purposes. pub client_version: String, /// Name of the node. Sent over the wire for debugging purposes. @@ -286,22 +335,26 @@ pub struct NetworkConfiguration { pub max_parallel_downloads: u32, } -impl Default for NetworkConfiguration { - fn default() -> Self { +impl NetworkConfiguration { + /// Create new default configuration + pub fn new, SV: Into>( + node_name: SN, + client_version: SV, + node_key: NodeKeyConfig, + net_config_path: &PathBuf, + ) -> Self { NetworkConfiguration { - config_path: None, - net_config_path: None, + net_config_path: net_config_path.clone(), listen_addresses: Vec::new(), public_addresses: Vec::new(), boot_nodes: Vec::new(), - node_key: NodeKeyConfig::Ed25519(Secret::New), + node_key, in_peers: 25, out_peers: 75, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Accept, - sentry_nodes: Vec::new(), - client_version: "unknown".into(), - node_name: "unknown".into(), + client_version: client_version.into(), + node_name: node_name.into(), transport: TransportConfig::Normal { enable_mdns: false, allow_private_ipv4: true, @@ -314,30 +367,39 @@ impl Default for NetworkConfiguration { } impl NetworkConfiguration { - /// Create a new instance of default settings. - pub fn new() -> Self { - Self::default() - } - /// Create new default configuration for localhost-only connection with random port (useful for testing) pub fn new_local() -> NetworkConfiguration { - let mut config = NetworkConfiguration::new(); + let mut config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + &std::env::current_dir().expect("current directory must exist"), + ); + config.listen_addresses = vec![ iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1))) .chain(iter::once(multiaddr::Protocol::Tcp(0))) .collect() ]; + config } /// Create new default configuration for localhost-only connection with random port (useful for testing) pub fn new_memory() -> NetworkConfiguration { - let mut config = NetworkConfiguration::new(); + let mut config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + &std::env::current_dir().expect("current directory must exist"), + ); + config.listen_addresses = vec![ iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1))) .chain(iter::once(multiaddr::Protocol::Tcp(0))) .collect() ]; + config } } @@ -402,6 +464,12 @@ pub enum NodeKeyConfig { Ed25519(Secret) } +impl Default for NodeKeyConfig { + fn default() -> NodeKeyConfig { + NodeKeyConfig::Ed25519(Secret::New) + } +} + /// The options for obtaining a Ed25519 secret key. pub type Ed25519Secret = Secret; diff --git a/client/network/src/debug_info.rs b/client/network/src/debug_info.rs index 17fb622f7cd3cab44f3b17ab508d7f1a3b258882..e2803cde35a772e5e094919ef31b364af640c487 100644 --- a/client/network/src/debug_info.rs +++ b/client/network/src/debug_info.rs @@ -17,14 +17,15 @@ use fnv::FnvHashMap; use futures::prelude::*; use libp2p::Multiaddr; -use libp2p::core::nodes::listeners::ListenerId; +use libp2p::core::connection::{ConnectionId, ListenerId}; use libp2p::core::{ConnectedPoint, either::EitherOutput, PeerId, PublicKey}; use libp2p::swarm::{IntoProtocolsHandler, IntoProtocolsHandlerSelect, ProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use libp2p::identify::{Identify, IdentifyEvent, IdentifyInfo}; use libp2p::ping::{Ping, PingConfig, PingEvent, PingSuccess}; use log::{debug, trace, error}; -use std::error; +use smallvec::SmallVec; +use std::{error, io}; use std::collections::hash_map::Entry; use std::pin::Pin; use std::task::{Context, Poll}; @@ -56,14 +57,27 @@ struct NodeInfo { /// When we will remove the entry about this node from the list, or `None` if we're connected /// to the node. info_expire: Option, - /// How we're connected to the node. - endpoint: ConnectedPoint, + /// Non-empty list of connected endpoints, one per connection. + endpoints: SmallVec<[ConnectedPoint; crate::MAX_CONNECTIONS_PER_PEER]>, /// Version reported by the remote, or `None` if unknown. client_version: Option, /// Latest ping time with this node. latest_ping: Option, } +impl NodeInfo { + fn new(endpoint: ConnectedPoint) -> Self { + let mut endpoints = SmallVec::new(); + endpoints.push(endpoint); + NodeInfo { + info_expire: None, + endpoints, + client_version: None, + latest_ping: None, + } + } +} + impl DebugInfoBehaviour { /// Builds a new `DebugInfoBehaviour`. pub fn new( @@ -121,9 +135,9 @@ impl DebugInfoBehaviour { pub struct Node<'a>(&'a NodeInfo); impl<'a> Node<'a> { - /// Returns the endpoint we are connected to or were last connected to. + /// Returns the endpoint of an established connection to the peer. pub fn endpoint(&self) -> &'a ConnectedPoint { - &self.0.endpoint + &self.0.endpoints[0] // `endpoints` are non-empty by definition } /// Returns the latest version information we know of. @@ -168,18 +182,17 @@ impl NetworkBehaviour for DebugInfoBehaviour { list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.ping.inject_connected(peer_id.clone(), endpoint.clone()); - self.identify.inject_connected(peer_id.clone(), endpoint.clone()); + fn inject_connected(&mut self, peer_id: &PeerId) { + self.ping.inject_connected(peer_id); + self.identify.inject_connected(peer_id); + } - match self.nodes_info.entry(peer_id) { + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.ping.inject_connection_established(peer_id, conn, endpoint); + self.identify.inject_connection_established(peer_id, conn, endpoint); + match self.nodes_info.entry(peer_id.clone()) { Entry::Vacant(e) => { - e.insert(NodeInfo { - info_expire: None, - endpoint, - client_version: None, - latest_ping: None, - }); + e.insert(NodeInfo::new(endpoint.clone())); } Entry::Occupied(e) => { let e = e.into_mut(); @@ -188,14 +201,26 @@ impl NetworkBehaviour for DebugInfoBehaviour { e.latest_ping = None; } e.info_expire = None; - e.endpoint = endpoint; + e.endpoints.push(endpoint.clone()); } } } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.ping.inject_disconnected(peer_id, endpoint.clone()); - self.identify.inject_disconnected(peer_id, endpoint); + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.ping.inject_connection_closed(peer_id, conn, endpoint); + self.identify.inject_connection_closed(peer_id, conn, endpoint); + + if let Some(entry) = self.nodes_info.get_mut(peer_id) { + entry.endpoints.retain(|ep| ep != endpoint) + } else { + error!(target: "sub-libp2p", + "Unknown connection to {:?} closed: {:?}", peer_id, endpoint); + } + } + + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.ping.inject_disconnected(peer_id); + self.identify.inject_disconnected(peer_id); if let Some(entry) = self.nodes_info.get_mut(peer_id) { entry.info_expire = Some(Instant::now() + CACHE_EXPIRE); @@ -205,26 +230,15 @@ impl NetworkBehaviour for DebugInfoBehaviour { } } - fn inject_node_event( + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent ) { match event { - EitherOutput::First(event) => self.ping.inject_node_event(peer_id, event), - EitherOutput::Second(event) => self.identify.inject_node_event(peer_id, event), - } - } - - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.ping.inject_replaced(peer_id.clone(), closed_endpoint.clone(), new_endpoint.clone()); - self.identify.inject_replaced(peer_id.clone(), closed_endpoint, new_endpoint.clone()); - - if let Some(entry) = self.nodes_info.get_mut(&peer_id) { - entry.endpoint = new_endpoint; - } else { - error!(target: "sub-libp2p", - "Disconnected from node we were not connected to {:?}", peer_id); + EitherOutput::First(event) => self.ping.inject_event(peer_id, connection, event), + EitherOutput::Second(event) => self.identify.inject_event(peer_id, connection, event), } } @@ -258,9 +272,9 @@ impl NetworkBehaviour for DebugInfoBehaviour { self.identify.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.ping.inject_listener_closed(id); - self.identify.inject_listener_closed(id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.ping.inject_listener_closed(id, reason); + self.identify.inject_listener_closed(id, reason); } fn poll( @@ -283,11 +297,12 @@ impl NetworkBehaviour for DebugInfoBehaviour { }, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, + handler, event: EitherOutput::First(event) }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => @@ -312,11 +327,12 @@ impl NetworkBehaviour for DebugInfoBehaviour { }, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, + handler, event: EitherOutput::Second(event) }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index ecce7d81e30cd62ed2856f03aed35eaa480ce215..a72c3cce65bd4a66a36bec3998920dc9f8578fc7 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -47,7 +47,7 @@ use futures::prelude::*; use futures_timer::Delay; -use libp2p::core::{nodes::listeners::ListenerId, ConnectedPoint, Multiaddr, PeerId, PublicKey}; +use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint, Multiaddr, PeerId, PublicKey}; use libp2p::swarm::{ProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use libp2p::kad::{Kademlia, KademliaEvent, Quorum, Record}; use libp2p::kad::GetClosestPeersError; @@ -58,7 +58,7 @@ use libp2p::{swarm::toggle::Toggle}; use libp2p::mdns::{Mdns, MdnsEvent}; use libp2p::multiaddr::Protocol; use log::{debug, info, trace, warn, error}; -use std::{cmp, collections::VecDeque, time::Duration}; +use std::{cmp, collections::VecDeque, io, time::Duration}; use std::task::{Context, Poll}; use sp_core::hexdisplay::HexDisplay; @@ -149,6 +149,7 @@ impl DiscoveryBehaviour { /// If we didn't know this address before, also generates a `Discovered` event. pub fn add_known_address(&mut self, peer_id: PeerId, addr: Multiaddr) { if self.user_defined.iter().all(|(p, a)| *p != peer_id && *a != addr) { + self.kademlia.add_address(&peer_id, addr.clone()); self.discoveries.push_back(peer_id.clone()); self.user_defined.push((peer_id, addr)); } @@ -259,18 +260,22 @@ impl NetworkBehaviour for DiscoveryBehaviour { list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { self.num_connections += 1; - NetworkBehaviour::inject_connected(&mut self.kademlia, peer_id, endpoint) + NetworkBehaviour::inject_connection_established(&mut self.kademlia, peer_id, conn, endpoint) } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { + fn inject_connected(&mut self, peer_id: &PeerId) { + NetworkBehaviour::inject_connected(&mut self.kademlia, peer_id) + } + + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { self.num_connections -= 1; - NetworkBehaviour::inject_disconnected(&mut self.kademlia, peer_id, endpoint) + NetworkBehaviour::inject_connection_closed(&mut self.kademlia, peer_id, conn, endpoint) } - fn inject_replaced(&mut self, peer_id: PeerId, closed: ConnectedPoint, opened: ConnectedPoint) { - NetworkBehaviour::inject_replaced(&mut self.kademlia, peer_id, closed, opened) + fn inject_disconnected(&mut self, peer_id: &PeerId) { + NetworkBehaviour::inject_disconnected(&mut self.kademlia, peer_id) } fn inject_addr_reach_failure( @@ -282,18 +287,19 @@ impl NetworkBehaviour for DiscoveryBehaviour { NetworkBehaviour::inject_addr_reach_failure(&mut self.kademlia, peer_id, addr, error) } - fn inject_node_event( + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: ::OutEvent, ) { - NetworkBehaviour::inject_node_event(&mut self.kademlia, peer_id, event) + NetworkBehaviour::inject_event(&mut self.kademlia, peer_id, connection, event) } fn inject_new_external_addr(&mut self, addr: &Multiaddr) { let new_addr = addr.clone() .with(Protocol::P2p(self.local_peer_id.clone().into())); - info!(target: "sub-libp2p", "Discovered new external address for our node: {}", new_addr); + info!(target: "sub-libp2p", "🔍 Discovered new external address for our node: {}", new_addr); NetworkBehaviour::inject_new_external_addr(&mut self.kademlia, addr) } @@ -315,9 +321,8 @@ impl NetworkBehaviour for DiscoveryBehaviour { NetworkBehaviour::inject_listener_error(&mut self.kademlia, id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - error!(target: "sub-libp2p", "Libp2p listener {:?} closed", id); - NetworkBehaviour::inject_listener_closed(&mut self.kademlia, id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + NetworkBehaviour::inject_listener_closed(&mut self.kademlia, id, reason); } fn poll( @@ -340,8 +345,9 @@ impl NetworkBehaviour for DiscoveryBehaviour { while let Poll::Ready(_) = self.next_kad_random_query.poll_unpin(cx) { let actually_started = if self.num_connections < self.discovery_only_if_under_num { let random_peer_id = PeerId::random(); - debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ - {:?}", random_peer_id); + debug!(target: "sub-libp2p", + "Libp2p <= Starting random Kademlia request for {:?}", + random_peer_id); self.kademlia.get_closest_peers(random_peer_id); true @@ -451,10 +457,10 @@ impl NetworkBehaviour for DiscoveryBehaviour { }, NetworkBehaviourAction::DialAddress { address } => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - NetworkBehaviourAction::SendEvent { peer_id, event } => - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }), + NetworkBehaviourAction::DialPeer { peer_id, condition } => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }), NetworkBehaviourAction::ReportObservedAddr { address } => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), } @@ -482,9 +488,9 @@ impl NetworkBehaviour for DiscoveryBehaviour { }, NetworkBehaviourAction::DialAddress { address } => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - NetworkBehaviourAction::SendEvent { event, .. } => + NetworkBehaviourAction::DialPeer { peer_id, condition } => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::NotifyHandler { event, .. } => match event {}, // `event` is an enum with no variant NetworkBehaviourAction::ReportObservedAddr { address } => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index f29fb00a4fdf1d0258302607d1be2ab70cfff7a1..d8afa1f1530ef8935df8a7177cd4d696d0c4adc2 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -90,23 +90,57 @@ //! ## Substreams //! //! Once a connection has been established and uses multiplexing, substreams can be opened. When -//! a substream is open, the **multistream-select** protocol is used to negotiate which protocol to -//! use on that given substream. In practice, Substrate opens the following substreams: -//! -//! - We periodically open an ephemeral substream in order to ping the remote and check whether the -//! connection is still alive. Failure for the remote to reply leads to a disconnection. This uses -//! the libp2p ping protocol. -//! - We periodically open an ephemeral substream in order to ask information from the remote. This -//! is called [the `identify` protocol](https://github.com/libp2p/specs/tree/master/identify). -//! - We periodically open ephemeral substreams for Kademlia random walk queries. Each Kademlia -//! query is done in a new separate substream. This uses the -//! [standard libp2p Kademlia protocol](https://github.com/libp2p/specs/pull/108). -//! - We optionally keep a substream alive for all Substrate-based communications. The name of the -//! protocol negotiated is based on the *protocol ID* passed as part of the network configuration. -//! This protocol ID should be unique for each chain and prevents nodes from different chains from -//! connecting to each other. More information below. -//! -//! ## The Substrate substream +//! a substream is open, the **multistream-select** protocol is used to negotiate which protocol +//! to use on that given substream. +//! +//! Protocols that are specific to a certain chain have a `` in their name. This +//! "protocol ID" is defined in the chain specifications. For example, the protocol ID of Polkadot +//! is "dot". In the protocol names below, `` must be replaced with the corresponding +//! protocol ID. +//! +//! > **Note**: It is possible for the same connection to be used for multiple chains. For example, +//! > one can use both the `/dot/sync/2` and `/sub/sync/2` protocols on the same +//! > connection, provided that the remote supports them. +//! +//! Substrate uses the following standard libp2p protocols: +//! +//! - **`/ipfs/ping/1.0.0`**. We periodically open an ephemeral substream in order to ping the +//! remote and check whether the connection is still alive. Failure for the remote to reply leads +//! to a disconnection. +//! - **[`/ipfs/id/1.0.0`](https://github.com/libp2p/specs/tree/master/identify)**. We +//! periodically open an ephemeral substream in order to ask information from the remote. +//! - **[`/ipfs/kad/1.0.0`](https://github.com/libp2p/specs/pull/108)**. We periodically open +//! ephemeral substreams for Kademlia random walk queries. Each Kademlia query is done in a +//! separate substream. +//! +//! Additionally, Substrate uses the following non-libp2p-standard protocols: +//! +//! - **`/substrate//`** (where `` must be replaced with the +//! protocol ID of the targeted chain, and `` is a number between 2 and 6). For each +//! connection we optionally keep an additional substream for all Substrate-based communications alive. +//! This protocol is considered legacy, and is progressively being replaced with alternatives. +//! This is designated as "The legacy Substrate substream" in this documentation. See below for +//! more details. +//! - **`//sync/2`** is a request-response protocol (see below) that lets one perform +//! requests for information about blocks. Each request is the encoding of a `BlockRequest` and +//! each response is the encoding of a `BlockResponse`, as defined in the `api.v1.proto` file in +//! this source tree. +//! - **`//light/2`** is a request-response protocol (see below) that lets one perform +//! light-client-related requests for information about the state. Each request is the encoding of +//! a `light::Request` and each response is the encoding of a `light::Response`, as defined in the +//! `light.v1.proto` file in this source tree. +//! - **`//transactions/1`** is a notifications protocol (see below) where +//! transactions are pushed to other nodes. The handshake is empty on both sides. The message +//! format is a SCALE-encoded list of transactions, where each transaction is an opaque list of +//! bytes. +//! - **`//block-announces/1`** is a notifications protocol (see below) where +//! block announces are pushed to other nodes. The handshake is empty on both sides. The message +//! format is a SCALE-encoded tuple containing a block header followed with an opaque list of +//! bytes containing some data associated with this block announcement, e.g. a candidate message. +//! - Notifications protocols that are registered using the `register_notifications_protocol` +//! method. For example: `/paritytech/grandpa/1`. See below for more information. +//! +//! ## The legacy Substrate substream //! //! Substrate uses a component named the **peerset manager (PSM)**. Through the discovery //! mechanism, the PSM is aware of the nodes that are part of the network and decides which nodes @@ -119,8 +153,8 @@ //! Note that at the moment there is no mechanism in place to solve the issues that arise where the //! two sides of a connection open the unique substream simultaneously. In order to not run into //! issues, only the dialer of a connection is allowed to open the unique substream. When the -//! substream is closed, the entire connection is closed as well. This is a bug, and should be -//! fixed by improving the protocol. +//! substream is closed, the entire connection is closed as well. This is a bug that will be +//! resolved by deprecating the protocol entirely. //! //! Within the unique Substrate substream, messages encoded using //! [*parity-scale-codec*](https://github.com/paritytech/parity-scale-codec) are exchanged. @@ -137,9 +171,46 @@ //! substream open with is chosen, and the information is requested from it. //! - Gossiping. Used for example by grandpa. //! -//! It is intended that in the future each of these components gets more isolated, so that they -//! are free to open and close their own substreams, and so that syncing and light client requests -//! are able to communicate with nodes outside of the range of the PSM. +//! ## Request-response protocols +//! +//! A so-called request-response protocol is defined as follow: +//! +//! - When a substream is opened, the opening side sends a message whose content is +//! protocol-specific. The message must be prefixed with an +//! [LEB128-encoded number](https://en.wikipedia.org/wiki/LEB128) indicating its length. After the +//! message has been sent, the writing side is closed. +//! - The remote sends back the response prefixed with a LEB128-encoded length, and closes its +//! side as well. +//! +//! Each request is performed in a new separate substream. +//! +//! ## Notifications protocols +//! +//! A so-called notifications protocol is defined as follow: +//! +//! - When a substream is opened, the opening side sends a handshake message whose content is +//! protocol-specific. The handshake message must be prefixed with an +//! [LEB128-encoded number](https://en.wikipedia.org/wiki/LEB128) indicating its length. The +//! handshake message can be of length 0, in which case the sender has to send a single `0`. +//! - The receiver then either immediately closes the substream, or answers with its own +//! LEB128-prefixed protocol-specific handshake response. The message can be of length 0, in which +//! case a single `0` has to be sent back. The receiver is then encouraged to close its sending +//! side. +//! - Once the handshake has completed, the notifications protocol is unidirectional. Only the +//! node which initiated the substream can push notifications. If the remote wants to send +//! notifications as well, it has to open its own undirectional substream. +//! - Each notification must be prefixed with an LEB128-encoded length. The encoding of the +//! messages is specific to each protocol. +//! +//! The API of `sc-network` allows one to register user-defined notification protocols. +//! `sc-network` automatically tries to open a substream towards each node for which the legacy +//! Substream substream is open. The handshake is then performed automatically. +//! +//! For example, the `sc-finality-grandpa` crate registers the `/paritytech/grandpa/1` +//! notifications protocol. +//! +//! At the moment, for backwards-compatibility, notification protocols are tied to the legacy +//! Substrate substream. In the future, though, it will no longer be the case. //! //! # Usage //! @@ -177,7 +248,7 @@ pub mod network_state; pub use service::{NetworkService, NetworkStateInfo, NetworkWorker, ExHashT, ReportHandle}; pub use protocol::PeerInfo; -pub use protocol::event::{Event, DhtEvent}; +pub use protocol::event::{Event, DhtEvent, ObservedRole}; pub use protocol::sync::SyncState; pub use libp2p::{Multiaddr, PeerId}; #[doc(inline)] @@ -185,13 +256,11 @@ pub use libp2p::multiaddr; pub use sc_peerset::ReputationChange; -/// Extension trait for `NetworkBehaviour` that also accepts discovering nodes. -trait DiscoveryNetBehaviour { - /// Notify the protocol that we have learned about the existence of nodes. - /// - /// Can (or most likely will) be called multiple times with the same `PeerId`s. - /// - /// Also note that there is no notification for expired nodes. The implementer must add a TTL - /// system, or remove nodes that will fail to reach. - fn add_discovered_nodes(&mut self, nodes: impl Iterator); -} +/// The maximum allowed number of established connections per peer. +/// +/// Typically, and by design of the network behaviours in this crate, +/// there is a single established connection per peer. However, to +/// avoid unnecessary and nondeterministic connection closure in +/// case of (possibly repeated) simultaneous dialing attempts between +/// two peers, the per-peer connection limit is not set to 1 but 2. +const MAX_CONNECTIONS_PER_PEER: usize = 2; diff --git a/client/network/src/on_demand_layer.rs b/client/network/src/on_demand_layer.rs index d672ed0b7f569507b8b94c1ae4766ab200938f9f..d881bf6fe243a7d5607fa9d3e0afba8760b517a5 100644 --- a/client/network/src/on_demand_layer.rs +++ b/client/network/src/on_demand_layer.rs @@ -16,16 +16,18 @@ //! On-demand requests service. -use crate::protocol::light_dispatch::RequestData; -use std::{collections::HashMap, pin::Pin, sync::Arc, task::Context, task::Poll}; -use futures::{prelude::*, channel::mpsc, channel::oneshot}; +use crate::protocol::light_client_handler; + +use futures::{channel::oneshot, prelude::*}; use parking_lot::Mutex; -use sp_blockchain::Error as ClientError; use sc_client_api::{ - Fetcher, FetchChecker, RemoteHeaderRequest, RemoteCallRequest, RemoteReadRequest, - RemoteChangesRequest, RemoteReadChildRequest, RemoteBodyRequest, + FetchChecker, Fetcher, RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest, + RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, StorageProof, ChangesProof, }; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sp_blockchain::Error as ClientError; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; +use std::{collections::HashMap, pin::Pin, sync::Arc, task::Context, task::Poll}; /// Implements the `Fetcher` trait of the client. Makes it possible for the light client to perform /// network requests for some state. @@ -41,18 +43,77 @@ pub struct OnDemand { /// Note that a better alternative would be to use a MPMC queue here, and add a `poll` method /// from the `OnDemand`. However there exists no popular implementation of MPMC channels in /// asynchronous Rust at the moment - requests_queue: Mutex>>>, + requests_queue: Mutex>>>, /// Sending side of `requests_queue`. - requests_send: mpsc::UnboundedSender>, + requests_send: TracingUnboundedSender>, } -impl OnDemand where +/// Dummy implementation of `FetchChecker` that always assumes that responses are bad. +/// +/// Considering that it is the responsibility of the client to build the fetcher, it can use this +/// implementation if it knows that it will never perform any request. +#[derive(Default, Clone)] +pub struct AlwaysBadChecker; + +impl FetchChecker for AlwaysBadChecker { + fn check_header_proof( + &self, + _request: &RemoteHeaderRequest, + _remote_header: Option, + _remote_proof: StorageProof, + ) -> Result { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } + + fn check_read_proof( + &self, + _request: &RemoteReadRequest, + _remote_proof: StorageProof, + ) -> Result,Option>>, ClientError> { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } + + fn check_read_child_proof( + &self, + _request: &RemoteReadChildRequest, + _remote_proof: StorageProof, + ) -> Result, Option>>, ClientError> { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } + + fn check_execution_proof( + &self, + _request: &RemoteCallRequest, + _remote_proof: StorageProof, + ) -> Result, ClientError> { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } + + fn check_changes_proof( + &self, + _request: &RemoteChangesRequest, + _remote_proof: ChangesProof + ) -> Result, u32)>, ClientError> { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } + + fn check_body_proof( + &self, + _request: &RemoteBodyRequest, + _body: Vec + ) -> Result, ClientError> { + Err(ClientError::Msg("AlwaysBadChecker".into())) + } +} + +impl OnDemand +where B::Header: HeaderT, { /// Creates new on-demand service. pub fn new(checker: Arc>) -> Self { - let (requests_send, requests_queue) = mpsc::unbounded(); + let (requests_send, requests_queue) = tracing_unbounded("mpsc_ondemand"); let requests_queue = Mutex::new(Some(requests_queue)); OnDemand { @@ -74,12 +135,15 @@ impl OnDemand where /// /// If this function returns `None`, that means that the receiver has already been extracted in /// the past, and therefore that something already handles the requests. - pub(crate) fn extract_receiver(&self) -> Option>> { + pub(crate) fn extract_receiver(&self) + -> Option>> + { self.requests_queue.lock().take() } } -impl Fetcher for OnDemand where +impl Fetcher for OnDemand +where B: BlockT, B::Header: HeaderT, { @@ -91,40 +155,55 @@ impl Fetcher for OnDemand where fn remote_header(&self, request: RemoteHeaderRequest) -> Self::RemoteHeaderResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteHeader(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::Header { request, sender }); RemoteResponse { receiver } } fn remote_read(&self, request: RemoteReadRequest) -> Self::RemoteReadResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteRead(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::Read { request, sender }); RemoteResponse { receiver } } fn remote_read_child( &self, - request: RemoteReadChildRequest + request: RemoteReadChildRequest, ) -> Self::RemoteReadResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteReadChild(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::ReadChild { request, sender }); RemoteResponse { receiver } } fn remote_call(&self, request: RemoteCallRequest) -> Self::RemoteCallResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteCall(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::Call { request, sender }); RemoteResponse { receiver } } - fn remote_changes(&self, request: RemoteChangesRequest) -> Self::RemoteChangesResult { + fn remote_changes( + &self, + request: RemoteChangesRequest, + ) -> Self::RemoteChangesResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteChanges(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::Changes { request, sender }); RemoteResponse { receiver } } fn remote_body(&self, request: RemoteBodyRequest) -> Self::RemoteBodyResult { let (sender, receiver) = oneshot::channel(); - let _ = self.requests_send.unbounded_send(RequestData::RemoteBody(request, sender)); + let _ = self + .requests_send + .unbounded_send(light_client_handler::Request::Body { request, sender }); RemoteResponse { receiver } } } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 7cf568065e873f14d4fcabc0af6aecf89b65fda2..365cbcbc5286f2efc643d3d563e06b2dc223b995 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -14,13 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::{DiscoveryNetBehaviour, config::ProtocolId}; +use crate::config::ProtocolId; use crate::utils::interval; use bytes::{Bytes, BytesMut}; use futures::prelude::*; use generic_proto::{GenericProto, GenericProtoOut}; use libp2p::{Multiaddr, PeerId}; -use libp2p::core::{ConnectedPoint, nodes::listeners::ListenerId}; +use libp2p::core::{ConnectedPoint, connection::{ConnectionId, ListenerId}}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use sp_core::{ @@ -38,21 +38,20 @@ use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, One, Zero, CheckedSub }; use sp_arithmetic::traits::SaturatedConversion; -use message::{BlockAnnounce, BlockAttributes, Direction, FromBlock, Message, RequestId}; -use message::generic::Message as GenericMessage; -use light_dispatch::{LightDispatch, LightDispatchNetwork, RequestData}; -use prometheus_endpoint::{Registry, Gauge, GaugeVec, PrometheusError, Opts, register, U64}; +use message::{BlockAnnounce, Message}; +use message::generic::{Message as GenericMessage, ConsensusMessage, Roles}; +use prometheus_endpoint::{Registry, Gauge, GaugeVec, HistogramVec, PrometheusError, Opts, register, U64}; use sync::{ChainSync, SyncState}; use crate::service::{TransactionPool, ExHashT}; -use crate::config::{BoxFinalityProofRequestBuilder, Roles}; +use crate::config::BoxFinalityProofRequestBuilder; use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::fmt::Write; -use std::{cmp, num::NonZeroUsize, pin::Pin, task::Poll, time}; +use std::{cmp, io, num::NonZeroUsize, pin::Pin, task::Poll, time}; use log::{log, Level, trace, debug, warn, error}; use crate::chain::{Client, FinalityProofProvider}; -use sc_client_api::{FetchChecker, ChangesProof, StorageProof}; +use sc_client_api::{ChangesProof, StorageProof}; use crate::error; use util::LruHashSet; use wasm_timer::Instant; @@ -74,7 +73,6 @@ pub mod block_requests; pub mod message; pub mod event; pub mod light_client_handler; -pub mod light_dispatch; pub mod sync; pub use block_requests::BlockRequests; @@ -201,9 +199,9 @@ pub struct Protocol { tick_timeout: Pin + Send>>, /// Interval at which we call `propagate_extrinsics`. propagate_timeout: Pin + Send>>, + /// Pending list of messages to return from `poll` as a priority. + pending_messages: VecDeque>, config: ProtocolConfig, - /// Handler for light client requests. - light_dispatch: LightDispatch, genesis_hash: B::Hash, sync: ChainSync, context_data: ContextData, @@ -221,8 +219,12 @@ pub struct Protocol { behaviour: GenericProto, /// For each legacy gossiping engine ID, the corresponding new protocol name. protocol_name_by_engine: HashMap>, - /// For each protocol name, the legacy gossiping engine ID. - protocol_engine_by_name: HashMap, ConsensusEngineId>, + /// For each protocol name, the legacy equivalent. + legacy_equiv_by_name: HashMap, Fallback>, + /// Name of the protocol used for transactions. + transactions_protocol: Cow<'static, [u8]>, + /// Name of the protocol used for block announces. + block_announces_protocol: Cow<'static, [u8]>, /// Prometheus metrics. metrics: Option, /// The `PeerId`'s of all boot nodes. @@ -272,132 +274,6 @@ pub struct PeerInfo { pub best_number: ::Number, } -struct LightDispatchIn<'a> { - behaviour: &'a mut GenericProto, - peerset: sc_peerset::PeersetHandle, -} - -impl<'a, B: BlockT> LightDispatchNetwork for LightDispatchIn<'a> { - fn report_peer(&mut self, who: &PeerId, reputation: sc_peerset::ReputationChange) { - self.peerset.report_peer(who.clone(), reputation) - } - - fn disconnect_peer(&mut self, who: &PeerId) { - self.behaviour.disconnect_peer(who) - } - - fn send_header_request(&mut self, who: &PeerId, id: RequestId, block: <::Header as HeaderT>::Number) { - let message: Message = message::generic::Message::RemoteHeaderRequest(message::RemoteHeaderRequest { - id, - block, - }); - - self.behaviour.send_packet(who, message.encode()) - } - - fn send_read_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - keys: Vec>, - ) { - let message: Message = message::generic::Message::RemoteReadRequest(message::RemoteReadRequest { - id, - block, - keys, - }); - - self.behaviour.send_packet(who, message.encode()) - } - - fn send_read_child_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - storage_key: Vec, - child_info: Vec, - child_type: u32, - keys: Vec>, - ) { - let message: Message = message::generic::Message::RemoteReadChildRequest(message::RemoteReadChildRequest { - id, - block, - storage_key, - child_info, - child_type, - keys, - }); - - self.behaviour.send_packet(who, message.encode()) - } - - fn send_call_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - method: String, - data: Vec - ) { - let message: Message = message::generic::Message::RemoteCallRequest(message::RemoteCallRequest { - id, - block, - method, - data, - }); - - self.behaviour.send_packet(who, message.encode()) - } - - fn send_changes_request( - &mut self, - who: &PeerId, - id: RequestId, - first: ::Hash, - last: ::Hash, - min: ::Hash, - max: ::Hash, - storage_key: Option>, - key: Vec, - ) { - let message: Message = message::generic::Message::RemoteChangesRequest(message::RemoteChangesRequest { - id, - first, - last, - min, - max, - storage_key, - key, - }); - - self.behaviour.send_packet(who, message.encode()) - } - - fn send_body_request( - &mut self, - who: &PeerId, - id: RequestId, - fields: BlockAttributes, - from: FromBlock<::Hash, <::Header as HeaderT>::Number>, - to: Option<::Hash>, - direction: Direction, - max: Option - ) { - let message: Message = message::generic::Message::BlockRequest(message::BlockRequest:: { - id, - fields, - from, - to, - direction, - max, - }); - - self.behaviour.send_packet(who, message.encode()) - } -} - /// Data necessary to create a context. struct ContextData { // All connected peers @@ -424,12 +300,22 @@ impl Default for ProtocolConfig { } } +/// Fallback mechanism to use to send a notification if no substream is open. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Fallback { + /// Use a `Message::Consensus` with the given engine ID. + Consensus(ConsensusEngineId), + /// The message is the bytes encoding of a `Transactions` (which is itself defined as a `Vec`). + Transactions, + /// The message is the bytes encoding of a `BlockAnnounce`. + BlockAnnounce, +} + impl Protocol { /// Create a new instance. pub fn new( config: ProtocolConfig, chain: Arc>, - checker: Arc>, transaction_pool: Arc>, finality_proof_provider: Option>>, finality_proof_request_builder: Option>, @@ -438,6 +324,7 @@ impl Protocol { block_announce_validator: Box + Send>, metrics_registry: Option<&Registry>, boot_node_ids: Arc>, + queue_size_report: Option, ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { let info = chain.info(); let sync = ChainSync::new( @@ -451,7 +338,7 @@ impl Protocol { let important_peers = { let mut imp_p = HashSet::new(); - for reserved in &peerset_config.reserved_nodes { + for reserved in peerset_config.priority_groups.iter().flat_map(|(_, l)| l.iter()) { imp_p.insert(reserved.clone()); } imp_p.shrink_to_fit(); @@ -460,18 +347,38 @@ impl Protocol { let (peerset, peerset_handle) = sc_peerset::Peerset::from_config(peerset_config); let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::>(); - let behaviour = GenericProto::new(protocol_id, versions, peerset); + let mut behaviour = GenericProto::new(protocol_id.clone(), versions, peerset, queue_size_report); + + let mut legacy_equiv_by_name = HashMap::new(); + + let transactions_protocol: Cow<'static, [u8]> = Cow::from({ + let mut proto = b"/".to_vec(); + proto.extend(protocol_id.as_bytes()); + proto.extend(b"/transactions/1"); + proto + }); + behaviour.register_notif_protocol(transactions_protocol.clone(), Vec::new()); + legacy_equiv_by_name.insert(transactions_protocol.clone(), Fallback::Transactions); + + let block_announces_protocol: Cow<'static, [u8]> = Cow::from({ + let mut proto = b"/".to_vec(); + proto.extend(protocol_id.as_bytes()); + proto.extend(b"/block-announces/1"); + proto + }); + behaviour.register_notif_protocol(block_announces_protocol.clone(), Vec::new()); + legacy_equiv_by_name.insert(block_announces_protocol.clone(), Fallback::BlockAnnounce); let protocol = Protocol { tick_timeout: Box::pin(interval(TICK_TIMEOUT)), propagate_timeout: Box::pin(interval(PROPAGATE_TIMEOUT)), + pending_messages: VecDeque::new(), config, context_data: ContextData { peers: HashMap::new(), stats: HashMap::new(), chain, }, - light_dispatch: LightDispatch::new(checker), genesis_hash: info.genesis_hash, sync, handshaking_peers: HashMap::new(), @@ -481,7 +388,9 @@ impl Protocol { peerset_handle: peerset_handle.clone(), behaviour, protocol_name_by_engine: HashMap::new(), - protocol_engine_by_name: HashMap::new(), + legacy_equiv_by_name, + transactions_protocol, + block_announces_protocol, metrics: if let Some(r) = metrics_registry { Some(Metrics::register(r)?) } else { @@ -572,20 +481,6 @@ impl Protocol { self.sync.num_sync_requests() } - /// Starts a new data demand request. - /// - /// The parameter contains a `Sender` where the result, once received, must be sent. - pub(crate) fn add_light_client_request(&mut self, rq: RequestData) { - self.light_dispatch.add_request(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, rq); - } - - fn is_light_response(&self, who: &PeerId, response_id: message::RequestId) -> bool { - self.light_dispatch.is_light_response(&who, response_id) - } - fn handle_response( &mut self, who: PeerId, @@ -645,15 +540,10 @@ impl Protocol { GenericMessage::Status(s) => return self.on_status_message(who, s), GenericMessage::BlockRequest(r) => self.on_block_request(who, r), GenericMessage::BlockResponse(r) => { - // Note, this is safe because only `ordinary bodies` and `remote bodies` are received in this matter. - if self.is_light_response(&who, r.id) { - self.on_remote_body_response(who, r); - } else { - if let Some(request) = self.handle_response(who.clone(), &r) { - let outcome = self.on_block_response(who.clone(), request, r); - self.update_peer_info(&who); - return outcome - } + if let Some(request) = self.handle_response(who.clone(), &r) { + let outcome = self.on_block_response(who.clone(), request, r); + self.update_peer_info(&who); + return outcome } }, GenericMessage::BlockAnnounce(announce) => { @@ -664,20 +554,20 @@ impl Protocol { GenericMessage::Transactions(m) => self.on_extrinsics(who, m), GenericMessage::RemoteCallRequest(request) => self.on_remote_call_request(who, request), - GenericMessage::RemoteCallResponse(response) => - self.on_remote_call_response(who, response), + GenericMessage::RemoteCallResponse(_) => + warn!(target: "sub-libp2p", "Received unexpected RemoteCallResponse"), GenericMessage::RemoteReadRequest(request) => self.on_remote_read_request(who, request), - GenericMessage::RemoteReadResponse(response) => - self.on_remote_read_response(who, response), + GenericMessage::RemoteReadResponse(_) => + warn!(target: "sub-libp2p", "Received unexpected RemoteReadResponse"), GenericMessage::RemoteHeaderRequest(request) => self.on_remote_header_request(who, request), - GenericMessage::RemoteHeaderResponse(response) => - self.on_remote_header_response(who, response), + GenericMessage::RemoteHeaderResponse(_) => + warn!(target: "sub-libp2p", "Received unexpected RemoteHeaderResponse"), GenericMessage::RemoteChangesRequest(request) => self.on_remote_changes_request(who, request), - GenericMessage::RemoteChangesResponse(response) => - self.on_remote_changes_response(who, response), + GenericMessage::RemoteChangesResponse(_) => + warn!(target: "sub-libp2p", "Received unexpected RemoteChangesResponse"), GenericMessage::FinalityProofRequest(request) => self.on_finality_proof_request(who, request), GenericMessage::FinalityProofResponse(response) => @@ -731,12 +621,18 @@ impl Protocol { ); } - fn send_message(&mut self, who: &PeerId, message: Message) { + fn send_message( + &mut self, + who: &PeerId, + message: Option<(Cow<'static, [u8]>, Vec)>, + legacy: Message, + ) { send_message::( &mut self.behaviour, &mut self.context_data.stats, who, message, + legacy, ); } @@ -762,10 +658,6 @@ impl Protocol { }; if let Some(_peer_data) = removed { self.sync.peer_disconnected(peer.clone()); - self.light_dispatch.on_disconnect(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, &peer); // Notify all the notification protocols as closed. CustomMessageOutcome::NotificationStreamClosed { @@ -793,11 +685,7 @@ impl Protocol { } } - fn on_block_request( - &mut self, - peer: PeerId, - request: message::BlockRequest - ) { + fn on_block_request(&mut self, peer: PeerId, request: message::BlockRequest) { trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?} for {:?}", request.id, peer, @@ -874,7 +762,7 @@ impl Protocol { blocks: blocks, }; trace!(target: "sync", "Sending BlockResponse with {} blocks", response.blocks.len()); - self.send_message(&peer, GenericMessage::BlockResponse(response)) + self.send_message(&peer, None, GenericMessage::BlockResponse(response)) } /// Adjusts the reputation of a node. @@ -950,10 +838,6 @@ impl Protocol { /// > **Note**: This method normally doesn't have to be called except for testing purposes. pub fn tick(&mut self) { self.maintain_peers(); - self.light_dispatch.maintain_peers(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }); self.report_metrics() } @@ -1101,10 +985,7 @@ impl Protocol { }; let info = self.context_data.peers.get(&who).expect("We just inserted above; QED").info.clone(); - self.light_dispatch.on_connect(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who.clone(), status.roles, status.best_number); + self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who.clone(), status.best_number)); if info.roles.is_full() { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), @@ -1132,10 +1013,15 @@ impl Protocol { &mut self, target: PeerId, engine_id: ConsensusEngineId, - message: impl Into> + message: impl Into>, ) { if let Some(protocol_name) = self.protocol_name_by_engine.get(&engine_id) { - self.behaviour.write_notification(&target, engine_id, protocol_name.clone(), message); + let message = message.into(); + let fallback = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { + engine_id, + data: message.clone(), + }).encode(); + self.behaviour.write_notification(&target, protocol_name.clone(), message, fallback); } else { error!( target: "sub-libp2p", @@ -1147,31 +1033,24 @@ impl Protocol { /// Registers a new notifications protocol. /// - /// You are very strongly encouraged to call this method very early on. Any connection open - /// will retain the protocols that were registered then, and not any new one. - pub fn register_notifications_protocol( - &mut self, + /// While registering a protocol while we already have open connections is discouraged, we + /// nonetheless handle it by notifying that we opened channels with everyone. This function + /// returns a list of substreams to open as a result. + pub fn register_notifications_protocol<'a>( + &'a mut self, engine_id: ConsensusEngineId, protocol_name: impl Into>, - ) -> Vec { + ) -> impl ExactSizeIterator + 'a { let protocol_name = protocol_name.into(); if self.protocol_name_by_engine.insert(engine_id, protocol_name.clone()).is_some() { error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", protocol_name); } else { - self.behaviour.register_notif_protocol(protocol_name.clone(), engine_id, Vec::new()); - self.protocol_engine_by_name.insert(protocol_name, engine_id); + self.behaviour.register_notif_protocol(protocol_name.clone(), Vec::new()); + self.legacy_equiv_by_name.insert(protocol_name, Fallback::Consensus(engine_id)); } - // Registering a protocol while we already have open connections isn't great, but for now - // we handle it by notifying that we opened channels with everyone. self.context_data.peers.iter() - .map(|(peer_id, peer)| - event::Event::NotificationStreamOpened { - remote: peer_id.clone(), - engine_id, - roles: peer.info.roles, - }) - .collect() + .map(|(peer_id, peer)| (peer_id, peer.info.roles)) } /// Called when peer sends us new extrinsics @@ -1229,7 +1108,7 @@ impl Protocol { fn do_propagate_extrinsics( &mut self, extrinsics: &[(H, B::Extrinsic)], - ) -> HashMap> { + ) -> HashMap> { let mut propagated_to = HashMap::new(); for (who, peer) in self.context_data.peers.iter_mut() { // never send extrinsics to the light node @@ -1251,10 +1130,12 @@ impl Protocol { .push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); + let encoded = to_send.encode(); send_message:: ( &mut self.behaviour, &mut self.context_data.stats, &who, + Some((self.transactions_protocol.clone(), encoded)), GenericMessage::Transactions(to_send) ) } @@ -1309,7 +1190,7 @@ impl Protocol { trace!(target: "sync", "Announcing block {:?} to {}", hash, who); let inserted = peer.known_blocks.insert(hash); if inserted || force { - let message: Message = GenericMessage::BlockAnnounce(message::BlockAnnounce { + let message = message::BlockAnnounce { header: header.clone(), state: if peer.info.protocol_version >= 4 { if is_best { @@ -1325,13 +1206,16 @@ impl Protocol { } else { None }, - }); + }; + + let encoded = message.encode(); send_message:: ( &mut self.behaviour, &mut self.context_data.stats, &who, - message, + Some((self.block_announces_protocol.clone(), encoded)), + Message::::BlockAnnounce(message), ) } } @@ -1350,18 +1234,20 @@ impl Protocol { chain_status: Vec::new(), // TODO: find a way to make this backwards-compatible }; - self.send_message(&who, GenericMessage::Status(status)) + self.send_message(&who, None, GenericMessage::Status(status)) } - fn on_block_announce(&mut self, who: PeerId, announce: BlockAnnounce) -> CustomMessageOutcome { + fn on_block_announce( + &mut self, + who: PeerId, + announce: BlockAnnounce, + ) -> CustomMessageOutcome { let hash = announce.header.hash(); + let number = *announce.header.number(); + if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) { peer.known_blocks.insert(hash.clone()); } - self.light_dispatch.update_best_number(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who.clone(), *announce.header.number()); let is_their_best = match announce.state.unwrap_or(message::BlockState::Best) { message::BlockState::Best => true, @@ -1376,7 +1262,11 @@ impl Protocol { // 1) we're on light client; // AND // 2) parent block is already imported and not pruned. - return CustomMessageOutcome::None + if is_their_best { + return CustomMessageOutcome::PeerNewBest(who, number); + } else { + return CustomMessageOutcome::None; + } } sync::OnBlockAnnounce::ImportHeader => () // We proceed with the import. } @@ -1401,33 +1291,37 @@ impl Protocol { }, ); match blocks_to_import { - Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), + Ok(sync::OnBlockData::Import(origin, blocks)) => { + if is_their_best { + self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who, number)); + } + CustomMessageOutcome::BlockImport(origin, blocks) + }, Ok(sync::OnBlockData::Request(peer, req)) => { self.send_request(&peer, GenericMessage::BlockRequest(req)); - CustomMessageOutcome::None + if is_their_best { + CustomMessageOutcome::PeerNewBest(who, number) + } else { + CustomMessageOutcome::None + } } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); self.peerset_handle.report_peer(id, repu); - CustomMessageOutcome::None + if is_their_best { + CustomMessageOutcome::PeerNewBest(who, number) + } else { + CustomMessageOutcome::None + } } } } - /// Call this when a block has been imported in the import queue and we should announce it on - /// the network. - pub fn on_block_imported(&mut self, header: &B::Header, data: Vec, is_best: bool) { + /// Call this when a block has been imported in the import queue + pub fn on_block_imported(&mut self, header: &B::Header, is_best: bool) { if is_best { self.sync.update_chain_info(header); } - - // blocks are not announced by light clients - if self.config.roles.is_light() { - return; - } - - // send out block announcements - self.send_announcement(header, data, is_best, false); } /// Call this when a block has been finalized. The sync layer may have some additional @@ -1468,6 +1362,7 @@ impl Protocol { self.send_message( &who, + None, GenericMessage::RemoteCallResponse(message::RemoteCallResponse { id: request.id, proof, @@ -1537,6 +1432,13 @@ impl Protocol { self.sync.request_finality_proof(&hash, number) } + /// Notify the protocol that we have learned about the existence of nodes. + /// + /// Can be called multiple times with the same `PeerId`s. + pub fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { + self.behaviour.add_discovered_nodes(peer_ids) + } + pub fn finality_proof_import_result( &mut self, request_block: (B::Hash, NumberFor), @@ -1545,18 +1447,6 @@ impl Protocol { self.sync.on_finality_proof_import(request_block, finalization_result) } - fn on_remote_call_response( - &mut self, - who: PeerId, - response: message::RemoteCallResponse - ) { - trace!(target: "sync", "Remote call response {} from {}", response.id, who); - self.light_dispatch.on_remote_call_response(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who, response); - } - fn on_remote_read_request( &mut self, who: PeerId, @@ -1598,6 +1488,7 @@ impl Protocol { }; self.send_message( &who, + None, GenericMessage::RemoteReadResponse(message::RemoteReadResponse { id: request.id, proof, @@ -1662,6 +1553,7 @@ impl Protocol { }; self.send_message( &who, + None, GenericMessage::RemoteReadResponse(message::RemoteReadResponse { id: request.id, proof, @@ -1669,18 +1561,6 @@ impl Protocol { ); } - fn on_remote_read_response( - &mut self, - who: PeerId, - response: message::RemoteReadResponse - ) { - trace!(target: "sync", "Remote read response {} from {}", response.id, who); - self.light_dispatch.on_remote_read_response(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who, response); - } - fn on_remote_header_request( &mut self, who: PeerId, @@ -1702,6 +1582,7 @@ impl Protocol { }; self.send_message( &who, + None, GenericMessage::RemoteHeaderResponse(message::RemoteHeaderResponse { id: request.id, header, @@ -1710,18 +1591,6 @@ impl Protocol { ); } - fn on_remote_header_response( - &mut self, - who: PeerId, - response: message::RemoteHeaderResponse, - ) { - trace!(target: "sync", "Remote header proof response {} from {}", response.id, who); - self.light_dispatch.on_remote_header_response(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who, response); - } - fn on_remote_changes_request( &mut self, who: PeerId, @@ -1772,6 +1641,7 @@ impl Protocol { }; self.send_message( &who, + None, GenericMessage::RemoteChangesResponse(message::RemoteChangesResponse { id: request.id, max: proof.max_block, @@ -1782,22 +1652,6 @@ impl Protocol { ); } - fn on_remote_changes_response( - &mut self, - who: PeerId, - response: message::RemoteChangesResponse, B::Hash>, - ) { - trace!(target: "sync", "Remote changes proof response {} from {} (max={})", - response.id, - who, - response.max - ); - self.light_dispatch.on_remote_changes_response(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, who, response); - } - fn on_finality_proof_request( &mut self, who: PeerId, @@ -1822,6 +1676,7 @@ impl Protocol { }; self.send_message( &who, + None, GenericMessage::FinalityProofResponse(message::FinalityProofResponse { id: 0, block: request.block, @@ -1848,17 +1703,6 @@ impl Protocol { } } - fn on_remote_body_response( - &mut self, - peer: PeerId, - response: message::BlockResponse - ) { - self.light_dispatch.on_remote_body_response(LightDispatchIn { - behaviour: &mut self.behaviour, - peerset: self.peerset_handle.clone(), - }, peer, response); - } - fn format_stats(&self) -> String { let mut out = String::new(); for (id, stats) in &self.context_data.stats { @@ -1930,6 +1774,8 @@ pub enum CustomMessageOutcome { NotificationStreamClosed { remote: PeerId, protocols: Vec }, /// Messages have been received on one or more notifications protocols. NotificationsReceived { remote: PeerId, messages: Vec<(ConsensusEngineId, Bytes)> }, + /// Peer has a reported a new head of chain. + PeerNewBest(PeerId, NumberFor), None, } @@ -1951,20 +1797,25 @@ fn send_request( peer.block_request = Some((Instant::now(), r.clone())); } } - send_message::(behaviour, stats, who, message) + send_message::(behaviour, stats, who, None, message) } fn send_message( behaviour: &mut GenericProto, stats: &mut HashMap<&'static str, PacketStats>, who: &PeerId, - message: Message, + message: Option<(Cow<'static, [u8]>, Vec)>, + legacy_message: Message, ) { - let encoded = message.encode(); - let mut stats = stats.entry(message.id()).or_default(); + let encoded = legacy_message.encode(); + let mut stats = stats.entry(legacy_message.id()).or_default(); stats.bytes_out += encoded.len() as u64; stats.count_out += 1; - behaviour.send_packet(who, encoded); + if let Some((proto, msg)) = message { + behaviour.write_notification(who, proto, msg, encoded); + } else { + behaviour.send_packet(who, encoded); + } } impl NetworkBehaviour for Protocol { @@ -1979,20 +1830,29 @@ impl NetworkBehaviour for Protocol { self.behaviour.addresses_of_peer(peer_id) } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.behaviour.inject_connected(peer_id, endpoint) + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.behaviour.inject_connection_established(peer_id, conn, endpoint) } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.behaviour.inject_disconnected(peer_id, endpoint) + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.behaviour.inject_connection_closed(peer_id, conn, endpoint) } - fn inject_node_event( + fn inject_connected(&mut self, peer_id: &PeerId) { + self.behaviour.inject_connected(peer_id) + } + + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.behaviour.inject_disconnected(peer_id) + } + + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent, ) { - self.behaviour.inject_node_event(peer_id, event) + self.behaviour.inject_event(peer_id, connection, event) } fn poll( @@ -2005,6 +1865,10 @@ impl NetworkBehaviour for Protocol { Self::OutEvent > > { + if let Some(message) = self.pending_messages.pop_front() { + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)); + } + while let Poll::Ready(Some(())) = self.tick_timeout.poll_next_unpin(cx) { self.tick(); } @@ -2045,10 +1909,10 @@ impl NetworkBehaviour for Protocol { Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)) => ev, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }), + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), }; @@ -2061,8 +1925,39 @@ impl NetworkBehaviour for Protocol { GenericProtoOut::CustomProtocolClosed { peer_id, .. } => { self.on_peer_disconnected(peer_id.clone()) }, - GenericProtoOut::CustomMessage { peer_id, message } => + GenericProtoOut::LegacyMessage { peer_id, message } => self.on_custom_message(peer_id, message), + GenericProtoOut::Notification { peer_id, protocol_name, message } => + match self.legacy_equiv_by_name.get(&protocol_name) { + Some(Fallback::Consensus(engine_id)) => { + CustomMessageOutcome::NotificationsReceived { + remote: peer_id, + messages: vec![(*engine_id, message.freeze())], + } + } + Some(Fallback::Transactions) => { + if let Ok(m) = message::Transactions::decode(&mut message.as_ref()) { + self.on_extrinsics(peer_id, m); + } else { + warn!(target: "sub-libp2p", "Failed to decode transactions list"); + } + CustomMessageOutcome::None + } + Some(Fallback::BlockAnnounce) => { + if let Ok(announce) = message::BlockAnnounce::decode(&mut message.as_ref()) { + let outcome = self.on_block_announce(peer_id.clone(), announce); + self.update_peer_info(&peer_id); + outcome + } else { + warn!(target: "sub-libp2p", "Failed to decode block announce"); + CustomMessageOutcome::None + } + } + None => { + error!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name); + CustomMessageOutcome::None + } + } GenericProtoOut::Clogged { peer_id, messages } => { debug!(target: "sync", "{} clogging messages:", messages.len()); for msg in messages.into_iter().take(5) { @@ -2081,10 +1976,6 @@ impl NetworkBehaviour for Protocol { } } - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.behaviour.inject_replaced(peer_id, closed_endpoint, new_endpoint) - } - fn inject_addr_reach_failure( &mut self, peer_id: Option<&PeerId>, @@ -2114,14 +2005,8 @@ impl NetworkBehaviour for Protocol { self.behaviour.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.behaviour.inject_listener_closed(id); - } -} - -impl DiscoveryNetBehaviour for Protocol { - fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { - self.behaviour.add_discovered_nodes(peer_ids) + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.behaviour.inject_listener_closed(id, reason); } } @@ -2134,8 +2019,7 @@ impl Drop for Protocol { #[cfg(test)] mod tests { use crate::PeerId; - use crate::protocol::light_dispatch::AlwaysBadChecker; - use crate::config::{EmptyTransactionPool, Roles}; + use crate::config::EmptyTransactionPool; use super::{CustomMessageOutcome, Protocol, ProtocolConfig}; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; @@ -2148,12 +2032,8 @@ mod tests { let client = Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0); let (mut protocol, _) = Protocol::::new( - ProtocolConfig { - roles: Roles::FULL, - max_parallel_downloads: 10, - }, + ProtocolConfig::default(), client.clone(), - Arc::new(AlwaysBadChecker), Arc::new(EmptyTransactionPool), None, None, @@ -2163,11 +2043,12 @@ mod tests { out_peers: 10, bootnodes: Vec::new(), reserved_only: false, - reserved_nodes: Vec::new(), + priority_groups: Vec::new(), }, Box::new(DefaultBlockAnnounceValidator::new(client.clone())), None, Default::default(), + None, ).unwrap(); let dummy_peer_id = PeerId::random(); diff --git a/client/network/src/protocol/block_requests.rs b/client/network/src/protocol/block_requests.rs index 5a947c0b6b5854ce05e393ea563976bb85656592..6af5023d39fe6b20661a4883c0d8a47416091c4e 100644 --- a/client/network/src/protocol/block_requests.rs +++ b/client/network/src/protocol/block_requests.rs @@ -35,6 +35,7 @@ use libp2p::{ ConnectedPoint, Multiaddr, PeerId, + connection::ConnectionId, upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, upgrade::{DeniedUpgrade, read_one, write_one} }, @@ -43,6 +44,7 @@ use libp2p::{ NetworkBehaviour, NetworkBehaviourAction, OneShotHandler, + OneShotHandlerConfig, PollParameters, SubstreamProtocol } @@ -257,20 +259,27 @@ where max_request_len: self.config.max_request_len, protocol: self.config.protocol.clone(), }; - OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + let mut cfg = OneShotHandlerConfig::default(); + cfg.inactive_timeout = self.config.inactivity_timeout; + OneShotHandler::new(SubstreamProtocol::new(p), cfg) } fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { Vec::new() } - fn inject_connected(&mut self, _peer: PeerId, _info: ConnectedPoint) { + fn inject_connected(&mut self, _peer: &PeerId) { } - fn inject_disconnected(&mut self, _peer: &PeerId, _info: ConnectedPoint) { + fn inject_disconnected(&mut self, _peer: &PeerId) { } - fn inject_node_event(&mut self, peer: PeerId, Request(request, mut stream): Request) { + fn inject_event( + &mut self, + peer: PeerId, + connection: ConnectionId, + Request(request, mut stream): Request + ) { match self.on_block_request(&peer, &request) { Ok(res) => { log::trace!("enqueueing block response for peer {} with {} blocks", peer, res.blocks.len()); diff --git a/client/network/src/protocol/event.rs b/client/network/src/protocol/event.rs index 78490863be9c250ad78605f634c95670ab8bf9ce..637bf805b5024cb24e1694018e72940d327b8c47 100644 --- a/client/network/src/protocol/event.rs +++ b/client/network/src/protocol/event.rs @@ -17,7 +17,6 @@ //! Network event types. These are are not the part of the protocol, but rather //! events that happen on the network like DHT get/put results received. -use crate::config::Roles; use bytes::Bytes; use libp2p::core::PeerId; use libp2p::kad::record::Key; @@ -55,8 +54,8 @@ pub enum Event { remote: PeerId, /// The concerned protocol. Each protocol uses a different substream. engine_id: ConsensusEngineId, - /// Roles that the remote . - roles: Roles, + /// Role of the remote. + role: ObservedRole, }, /// Closed a substream with the given node. Always matches a corresponding previous @@ -76,3 +75,26 @@ pub enum Event { messages: Vec<(ConsensusEngineId, Bytes)>, }, } + +/// Role that the peer sent to us during the handshake, with the addition of what our local node +/// knows about that peer. +#[derive(Debug, Clone)] +pub enum ObservedRole { + /// Full node. + Full, + /// Light node. + Light, + /// When we are a validator node, this is a sentry that protects us. + OurSentry, + /// When we are a sentry node, this is the authority we are protecting. + OurGuardedAuthority, + /// Third-party authority. + Authority, +} + +impl ObservedRole { + /// Returns `true` for `ObservedRole::Light`. + pub fn is_light(&self) -> bool { + matches!(self, ObservedRole::Light) + } +} diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs index 727415baaf5bcd9a4b06ca847d9aeb6968f78ab1..b38a97cb8f537501716f3923b6cb1501d9d4f5c8 100644 --- a/client/network/src/protocol/generic_proto/behaviour.rs +++ b/client/network/src/protocol/generic_proto/behaviour.rs @@ -14,35 +14,40 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::{DiscoveryNetBehaviour, config::ProtocolId}; -use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; +use crate::config::ProtocolId; use crate::protocol::generic_proto::handler::{NotifsHandlerProto, NotifsHandlerOut, NotifsHandlerIn}; use crate::protocol::generic_proto::upgrade::RegisteredProtocol; use bytes::BytesMut; -use codec::Encode as _; use fnv::FnvHashMap; use futures::prelude::*; -use libp2p::core::{ConnectedPoint, Multiaddr, PeerId}; -use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; +use libp2p::core::{ConnectedPoint, Multiaddr, PeerId, connection::ConnectionId}; +use libp2p::swarm::{ + DialPeerCondition, + NetworkBehaviour, + NetworkBehaviourAction, + NotifyHandler, + PollParameters +}; use log::{debug, error, trace, warn}; +use prometheus_endpoint::HistogramVec; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; -use sp_runtime::ConsensusEngineId; -use std::{borrow::Cow, collections::hash_map::Entry, cmp}; -use std::{error, mem, pin::Pin, str, time::Duration}; use std::task::{Context, Poll}; +use std::{borrow::Cow, cmp, collections::hash_map::Entry}; +use std::{error, mem, pin::Pin, str, time::Duration}; use wasm_timer::Instant; -/// Network behaviour that handles opening substreams for custom protocols with other nodes. +/// Network behaviour that handles opening substreams for custom protocols with other peers. /// /// ## Legacy vs new protocol /// /// The `GenericProto` behaves as following: /// -/// - Whenever a connection is established, we open a single substream (called "legay protocol" in -/// the source code). This substream name depends on the `protocol_id` and `versions` passed at -/// initialization. If the remote refuses this substream, we close the connection. +/// - Whenever a connection is established, we open a single substream (called "legacy protocol" in +/// the source code) on that connection. This substream name depends on the `protocol_id` and +/// `versions` passed at initialization. If the remote refuses this substream, we close the +/// connection. /// /// - For each registered protocol, we also open an additional substream for this protocol. If the /// remote refuses this substream, then it's fine. @@ -57,34 +62,58 @@ use wasm_timer::Instant; /// /// - The libp2p swarm that opens new connections and reports disconnects. /// - The connection handler (see `handler.rs`) that handles individual connections. -/// - The peerset manager (PSM) that requests links to nodes to be established or broken. +/// - The peerset manager (PSM) that requests links to peers to be established or broken. /// - The external API, that requires knowledge of the links that have been established. /// /// Each connection handler can be in four different states: Enabled+Open, Enabled+Closed, /// Disabled+Open, or Disabled+Closed. The Enabled/Disabled component must be in sync with the /// peerset manager. For example, if the peerset manager requires a disconnection, we disable the -/// existing handler. The Open/Closed component must be in sync with the external API. +/// connection handlers of that peer. The Open/Closed component must be in sync with the external +/// API. +/// +/// However, a connection handler for a peer only exists if we are actually connected to that peer. +/// What this means is that there are six possible states for each peer: Disconnected, Dialing +/// (trying to connect), Enabled+Open, Enabled+Closed, Disabled+Open, Disabled+Closed. +/// Most notably, the Dialing state must correspond to a "link established" state in the peerset +/// manager. In other words, the peerset manager doesn't differentiate whether we are dialing a +/// peer or connected to it. /// -/// However a connection handler only exists if we are actually connected to a node. What this -/// means is that there are six possible states for each node: Disconnected, Dialing (trying to -/// reach it), Enabled+Open, Enabled+Closed, Disabled+open, Disabled+Closed. Most notably, the -/// Dialing state must correspond to a "link established" state in the peerset manager. In other -/// words, the peerset manager doesn't differentiate whether we are dialing a node or connected -/// to it. +/// There may be multiple connections to a peer. However, the status of a peer on +/// the API of this behaviour and towards the peerset manager is aggregated in +/// the following way: /// -/// Additionally, there also exists a "banning" system. If we fail to dial a node, we "ban" it for -/// a few seconds. If the PSM requests a node that is in the "banned" state, then we delay the -/// actual dialing attempt until after the ban expires, but the PSM will still consider the link -/// to be established. -/// Note that this "banning" system is not an actual ban. If a "banned" node tries to connect to -/// us, we accept the connection. The "banning" system is only about delaying dialing attempts. +/// 1. The enabled/disabled status is the same across all connections, as +/// decided by the peerset manager. +/// 2. `send_packet` and `write_notification` always send all data over +/// the same connection to preserve the ordering provided by the transport, +/// as long as that connection is open. If it closes, a second open +/// connection may take over, if one exists, but that case should be no +/// different than a single connection failing and being re-established +/// in terms of potential reordering and dropped messages. Messages can +/// be received on any connection. +/// 3. The behaviour reports `GenericProtoOut::CustomProtocolOpen` when the +/// first connection reports `NotifsHandlerOut::Open`. +/// 4. The behaviour reports `GenericProtoOut::CustomProtocolClosed` when the +/// last connection reports `NotifsHandlerOut::Closed`. +/// +/// In this way, the number of actual established connections to the peer is +/// an implementation detail of this behaviour. Note that, in practice and at +/// the time of this writing, there may be at most two connections to a peer +/// and only as a result of simultaneous dialing. However, the implementation +/// accommodates for any number of connections. +/// +/// Additionally, there also exists a "banning" system. If we fail to dial a peer, we "ban" it for +/// a few seconds. If the PSM requests connecting to a peer that is currently "banned", the next +/// dialing attempt is delayed until after the ban expires. However, the PSM will still consider +/// the peer to be connected. This "ban" is thus not a ban in a strict sense: If a "banned" peer +/// tries to connect, the connection is accepted. A ban only delays dialing attempts. /// pub struct GenericProto { /// Legacy protocol to open with peers. Never modified. legacy_protocol: RegisteredProtocol, /// Notification protocols. Entries are only ever added and not removed. - notif_protocols: Vec<(Cow<'static, [u8]>, ConsensusEngineId, Vec)>, + notif_protocols: Vec<(Cow<'static, [u8]>, Vec)>, /// Receiver for instructions about who to connect to or disconnect from. peerset: sc_peerset::Peerset, @@ -102,6 +131,9 @@ pub struct GenericProto { /// Events to produce from `poll()`. events: SmallVec<[NetworkBehaviourAction; 4]>, + + /// If `Some`, report the message queue sizes on this `Histogram`. + queue_size_report: Option, } /// State of a peer we're connected to. @@ -112,14 +144,14 @@ enum PeerState { /// the state machine code. Poisoned, - /// The peer misbehaved. If the PSM wants us to connect to this node, we will add an artificial + /// The peer misbehaved. If the PSM wants us to connect to this peer, we will add an artificial /// delay to the connection. Banned { - /// Until when the node is banned. + /// Until when the peer is banned. until: Instant, }, - /// The peerset requested that we connect to this peer. We are not connected to this node. + /// The peerset requested that we connect to this peer. We are currently not connected. PendingRequest { /// When to actually start dialing. timer: futures_timer::Delay, @@ -130,16 +162,13 @@ enum PeerState { /// The peerset requested that we connect to this peer. We are currently dialing this peer. Requested, - /// We are connected to this peer but the peerset refused it. This peer can still perform - /// Kademlia queries and such, but should get disconnected in a few seconds. + /// We are connected to this peer but the peerset refused it. + /// + /// We may still have ongoing traffic with that peer, but it should cease shortly. Disabled { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we still have a custom protocol open with it. It will likely get closed in - /// a short amount of time, but we need to keep the information in order to not have a - /// state mismatch. - open: bool, - /// If `Some`, the node is banned until the given `Instant`. + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, + /// If `Some`, any dial attempts to this peer are delayed until the given `Instant`. banned_until: Option, }, @@ -147,12 +176,8 @@ enum PeerState { /// will be enabled when `timer` fires. This peer can still perform Kademlia queries and such, /// but should get disconnected in a few seconds. DisabledPendingEnable { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we still have a custom protocol open with it. It will likely get closed in - /// a short amount of time, but we need to keep the information in order to not have a - /// state mismatch. - open: bool, + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, /// When to enable this remote. timer: futures_timer::Delay, /// When the `timer` will trigger. @@ -162,33 +187,41 @@ enum PeerState { /// We are connected to this peer and the peerset has accepted it. The handler is in the /// enabled state. Enabled { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we have a custom protocol open with this peer. - open: bool, + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, }, - /// We are connected to this peer, and we sent an incoming message to the peerset. The handler - /// is in initialization mode. We are waiting for the Accept or Reject from the peerset. There - /// is a corresponding entry in `incoming`. - Incoming { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - }, + /// We received an incoming connection from this peer and forwarded that + /// connection request to the peerset. The connection handlers are waiting + /// for initialisation, i.e. to be enabled or disabled based on whether + /// the peerset accepts or rejects the peer. + Incoming, } impl PeerState { - /// True if we have an open channel with that node. + /// True if there exists an established connection to tbe peer + /// that is open for custom protocol traffic. fn is_open(&self) -> bool { + self.get_open().is_some() + } + + /// Returns the connection ID of the first established connection + /// that is open for custom protocol traffic. + fn get_open(&self) -> Option { match self { - PeerState::Poisoned => false, - PeerState::Banned { .. } => false, - PeerState::PendingRequest { .. } => false, - PeerState::Requested => false, - PeerState::Disabled { open, .. } => *open, - PeerState::DisabledPendingEnable { open, .. } => *open, - PeerState::Enabled { open, .. } => *open, - PeerState::Incoming { .. } => false, + PeerState::Disabled { open, .. } | + PeerState::DisabledPendingEnable { open, .. } | + PeerState::Enabled { open, .. } => + if !open.is_empty() { + Some(open[0]) + } else { + None + } + PeerState::Poisoned => None, + PeerState::Banned { .. } => None, + PeerState::PendingRequest { .. } => None, + PeerState::Requested => None, + PeerState::Incoming { .. } => None, } } @@ -210,7 +243,7 @@ impl PeerState { /// State of an "incoming" message sent to the peer set manager. #[derive(Debug)] struct IncomingPeer { - /// Id of the node that is concerned. + /// Id of the remote peer of the incoming connection. peer_id: PeerId, /// If true, this "incoming" still corresponds to an actual connection. If false, then the /// connection corresponding to it has been closed or replaced already. @@ -224,10 +257,8 @@ struct IncomingPeer { pub enum GenericProtoOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { - /// Id of the node we have opened a connection with. + /// Id of the peer we are connected to. peer_id: PeerId, - /// Endpoint used for this custom protocol. - endpoint: ConnectedPoint, }, /// Closed a custom protocol with the remote. @@ -238,12 +269,22 @@ pub enum GenericProtoOut { reason: Cow<'static, str>, }, + /// Receives a message on the legacy substream. + LegacyMessage { + /// Id of the peer the message came from. + peer_id: PeerId, + /// Message that has been received. + message: BytesMut, + }, + /// Receives a message on a custom protocol substream. /// /// Also concerns received notifications for the notifications API. - CustomMessage { + Notification { /// Id of the peer the message came from. peer_id: PeerId, + /// Engine corresponding to the message. + protocol_name: Cow<'static, [u8]>, /// Message that has been received. message: BytesMut, }, @@ -260,10 +301,14 @@ pub enum GenericProtoOut { impl GenericProto { /// Creates a `CustomProtos`. + /// + /// The `queue_size_report` is an optional Prometheus metric that can report the size of the + /// messages queue. If passed, it must have one label for the protocol name. pub fn new( protocol: impl Into, versions: &[u8], peerset: sc_peerset::Peerset, + queue_size_report: Option, ) -> Self { let legacy_protocol = RegisteredProtocol::new(protocol, versions); @@ -275,6 +320,7 @@ impl GenericProto { incoming: SmallVec::new(), next_incoming_index: sc_peerset::IncomingIndex(0), events: SmallVec::new(), + queue_size_report, } } @@ -285,10 +331,9 @@ impl GenericProto { pub fn register_notif_protocol( &mut self, protocol_name: impl Into>, - engine_id: ConsensusEngineId, handshake_msg: impl Into> ) { - self.notif_protocols.push((protocol_name.into(), engine_id, handshake_msg.into())); + self.notif_protocols.push((protocol_name.into(), handshake_msg.into())); } /// Returns the number of discovered nodes that we keep in memory. @@ -301,7 +346,7 @@ impl GenericProto { self.peers.iter().filter(|(_, state)| state.is_open()).map(|(id, _)| id) } - /// Returns true if we have a channel open with this node. + /// Returns true if we have an open connection to the given peer. pub fn is_open(&self, peer_id: &PeerId) -> bool { self.peers.get(peer_id).map(|p| p.is_open()).unwrap_or(false) } @@ -312,8 +357,8 @@ impl GenericProto { self.disconnect_peer_inner(peer_id, None); } - /// Inner implementation of `disconnect_peer`. If `ban` is `Some`, we ban the node for the - /// specific duration. + /// Inner implementation of `disconnect_peer`. If `ban` is `Some`, we ban the peer + /// for the specific duration. fn disconnect_peer_inner(&mut self, peer_id: &PeerId, ban: Option) { let mut entry = if let Entry::Occupied(entry) = self.peers.entry(peer_id.clone()) { entry @@ -329,7 +374,11 @@ impl GenericProto { st @ PeerState::Banned { .. } => *entry.into_mut() = st, // DisabledPendingEnable => Disabled. - PeerState::DisabledPendingEnable { open, connected_point, timer_deadline, .. } => { + PeerState::DisabledPendingEnable { + open, + timer_deadline, + timer: _ + } => { debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); self.peerset.dropped(peer_id.clone()); let banned_until = Some(if let Some(ban) = ban { @@ -337,24 +386,31 @@ impl GenericProto { } else { timer_deadline }); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until + } }, // Enabled => Disabled. - PeerState::Enabled { open, connected_point } => { + PeerState::Enabled { open } => { debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); self.peerset.dropped(peer_id.clone()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until + } }, // Incoming => Disabled. - PeerState::Incoming { connected_point, .. } => { + PeerState::Incoming => { let inc = if let Some(inc) = self.incoming.iter_mut() .find(|i| i.peer_id == *entry.key() && i.alive) { inc @@ -366,12 +422,16 @@ impl GenericProto { inc.alive = false; debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); - *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open: SmallVec::new(), + banned_until + } }, PeerState::Poisoned => @@ -399,6 +459,16 @@ impl GenericProto { } } + /// Notify the behaviour that we have learned about the existence of nodes. + /// + /// Can be called multiple times with the same `PeerId`s. + pub fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { + self.peerset.discovered(peer_ids.into_iter().map(|peer_id| { + debug!(target: "sub-libp2p", "PSM <= Discovered({:?})", peer_id); + peer_id + })); + } + /// Sends a notification to a peer. /// /// Has no effect if the custom protocol is not open with the given peer. @@ -406,32 +476,40 @@ impl GenericProto { /// Also note that even if we have a valid open substream, it may in fact be already closed /// without us knowing, in which case the packet will not be received. /// - /// > **Note**: Ideally the `engine_id` parameter wouldn't be necessary. See the documentation - /// > of [`NotifsHandlerIn`] for more information. + /// The `fallback` parameter is used for backwards-compatibility reason if the remote doesn't + /// support our protocol. One needs to pass the equivalent of what would have been passed + /// with `send_packet`. pub fn write_notification( &mut self, target: &PeerId, - engine_id: ConsensusEngineId, protocol_name: Cow<'static, [u8]>, message: impl Into>, + encoded_fallback_message: Vec, ) { - if !self.is_open(target) { - return; - } + let conn = match self.peers.get(target).and_then(|p| p.get_open()) { + None => { + debug!(target: "sub-libp2p", + "Tried to sent notification to {:?} without an open channel.", + target); + return + }, + Some(conn) => conn + }; trace!( target: "sub-libp2p", - "External API => Notification for {:?} with protocol {:?}", + "External API => Notification({:?}, {:?})", target, str::from_utf8(&protocol_name) ); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), + handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendNotification { message: message.into(), - engine_id, + encoded_fallback_message, protocol_name, }, }); @@ -444,14 +522,21 @@ impl GenericProto { /// Also note that even we have a valid open substream, it may in fact be already closed /// without us knowing, in which case the packet will not be received. pub fn send_packet(&mut self, target: &PeerId, message: Vec) { - if !self.is_open(target) { - return; - } + let conn = match self.peers.get(target).and_then(|p| p.get_open()) { + None => { + debug!(target: "sub-libp2p", + "Tried to sent packet to {:?} without an open channel.", + target); + return + } + Some(conn) => conn + }; trace!(target: "sub-libp2p", "External API => Packet for {:?}", target); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), + handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendLegacy { message, } @@ -463,7 +548,7 @@ impl GenericProto { self.peerset.debug_info() } - /// Function that is called when the peerset wants us to connect to a node. + /// Function that is called when the peerset wants us to connect to a peer. fn peerset_report_connect(&mut self, peer_id: PeerId) { let mut occ_entry = match self.peers.entry(peer_id) { Entry::Occupied(entry) => entry, @@ -471,7 +556,10 @@ impl GenericProto { // If there's no entry in `self.peers`, start dialing. debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: entry.key().clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: entry.key().clone(), + condition: DialPeerCondition::Disconnected + }); entry.insert(PeerState::Requested); return; } @@ -492,36 +580,41 @@ impl GenericProto { PeerState::Banned { .. } => { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", occ_entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", occ_entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: occ_entry.key().clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: occ_entry.key().clone(), + condition: DialPeerCondition::Disconnected + }); *occ_entry.into_mut() = PeerState::Requested; }, - PeerState::Disabled { open, ref connected_point, banned_until: Some(ref banned) } - if *banned > now => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Has idle connection through \ - {:?} but node is banned until {:?}", occ_entry.key(), connected_point, banned); + PeerState::Disabled { + open, + banned_until: Some(ref banned) + } if *banned > now => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): But peer is banned until {:?}", + occ_entry.key(), banned); *occ_entry.into_mut() = PeerState::DisabledPendingEnable { - connected_point: connected_point.clone(), open, timer: futures_timer::Delay::new(banned.clone() - now), timer_deadline: banned.clone(), }; }, - PeerState::Disabled { open, connected_point, banned_until: _ } => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling previously-idle \ - connection through {:?}", occ_entry.key(), connected_point); + PeerState::Disabled { open, banned_until: _ } => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling connections.", + occ_entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *occ_entry.into_mut() = PeerState::Enabled { connected_point, open }; + *occ_entry.into_mut() = PeerState::Enabled { open }; }, - PeerState::Incoming { connected_point, .. } => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling incoming \ - connection through {:?}", occ_entry.key(), connected_point); + PeerState::Incoming => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling connections.", + occ_entry.key()); if let Some(inc) = self.incoming.iter_mut() .find(|i| i.peer_id == *occ_entry.key() && i.alive) { inc.alive = false; @@ -530,26 +623,30 @@ impl GenericProto { incoming for incoming peer") } debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *occ_entry.into_mut() = PeerState::Enabled { connected_point, open: false }; + *occ_entry.into_mut() = PeerState::Enabled { open: SmallVec::new() }; }, st @ PeerState::Enabled { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Already connected to this \ - peer", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Already connected.", + occ_entry.key()); *occ_entry.into_mut() = st; }, st @ PeerState::DisabledPendingEnable { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Already have an idle \ - connection to this peer and waiting to enable it", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Already pending enabling.", + occ_entry.key()); *occ_entry.into_mut() = st; }, st @ PeerState::Requested { .. } | st @ PeerState::PendingRequest { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Received a previous \ - request for that peer", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Duplicate request.", + occ_entry.key()); *occ_entry.into_mut() = st; }, @@ -558,55 +655,63 @@ impl GenericProto { } } - /// Function that is called when the peerset wants us to disconnect from a node. + /// Function that is called when the peerset wants us to disconnect from a peer. fn peerset_report_disconnect(&mut self, peer_id: PeerId) { let mut entry = match self.peers.entry(peer_id) { Entry::Occupied(entry) => entry, Entry::Vacant(entry) => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Node already disabled", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Already disabled.", entry.key()); return } }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { st @ PeerState::Disabled { .. } | st @ PeerState::Banned { .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Node already disabled", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Already disabled.", entry.key()); *entry.into_mut() = st; }, - PeerState::DisabledPendingEnable { open, connected_point, timer_deadline, .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Interrupting pending \ - enable", entry.key()); + PeerState::DisabledPendingEnable { + open, + timer_deadline, + timer: _ + } => { + debug!(target: "sub-libp2p", + "PSM => Drop({:?}): Interrupting pending enabling.", + entry.key()); *entry.into_mut() = PeerState::Disabled { open, - connected_point, banned_until: Some(timer_deadline), }; }, - PeerState::Enabled { open, connected_point } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Disabling connection", entry.key()); + PeerState::Enabled { open } => { + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Disabling connections.", entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until: None } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until: None + } }, - st @ PeerState::Incoming { .. } => { - error!(target: "sub-libp2p", "PSM => Drop({:?}): Was in incoming mode", + st @ PeerState::Incoming => { + error!(target: "sub-libp2p", "PSM => Drop({:?}): Not enabled (Incoming).", entry.key()); *entry.into_mut() = st; }, PeerState::Requested => { // We don't cancel dialing. Libp2p doesn't expose that on purpose, as other - // sub-systems (such as the discovery mechanism) may require dialing this node as + // sub-systems (such as the discovery mechanism) may require dialing this peer as // well at the same time. - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Was not yet connected", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Not yet connected.", entry.key()); entry.remove(); }, PeerState::PendingRequest { timer_deadline, .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Was not yet connected", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Not yet connected", entry.key()); *entry.into_mut() = PeerState::Banned { until: timer_deadline } }, @@ -615,7 +720,8 @@ impl GenericProto { } } - /// Function that is called when the peerset wants us to accept an incoming node. + /// Function that is called when the peerset wants us to accept a connection + /// request from a peer. fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -632,34 +738,25 @@ impl GenericProto { return } - let state = if let Some(state) = self.peers.get_mut(&incoming.peer_id) { - state - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in peers \ - corresponding to an alive incoming"); - return - }; - - let connected_point = if let PeerState::Incoming { connected_point } = state { - connected_point.clone() - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: entry in peers corresponding \ - to an alive incoming is not in incoming state"); - return - }; - - debug!(target: "sub-libp2p", "PSM => Accept({:?}, {:?}): Enabling connection \ - through {:?}", index, incoming.peer_id, connected_point); - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { - peer_id: incoming.peer_id, - event: NotifsHandlerIn::Enable, - }); - - *state = PeerState::Enabled { open: false, connected_point }; + match self.peers.get_mut(&incoming.peer_id) { + Some(state @ PeerState::Incoming) => { + debug!(target: "sub-libp2p", "PSM => Accept({:?}, {:?}): Enabling connections.", + index, incoming.peer_id); + debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: incoming.peer_id, + handler: NotifyHandler::All, + event: NotifsHandlerIn::Enable, + }); + *state = PeerState::Enabled { open: SmallVec::new() }; + } + peer => error!(target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", + peer) + } } - /// Function that is called when the peerset wants us to reject an incoming node. + /// Function that is called when the peerset wants us to reject an incoming peer. fn peerset_report_reject(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -674,39 +771,25 @@ impl GenericProto { return } - let state = if let Some(state) = self.peers.get_mut(&incoming.peer_id) { - state - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in peers \ - corresponding to an alive incoming"); - return - }; - - let connected_point = if let PeerState::Incoming { connected_point } = state { - connected_point.clone() - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: entry in peers corresponding \ - to an alive incoming is not in incoming state"); - return - }; - - debug!(target: "sub-libp2p", "PSM => Reject({:?}, {:?}): Rejecting connection through \ - {:?}", index, incoming.peer_id, connected_point); - debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { - peer_id: incoming.peer_id, - event: NotifsHandlerIn::Disable, - }); - *state = PeerState::Disabled { open: false, connected_point, banned_until: None }; - } -} - -impl DiscoveryNetBehaviour for GenericProto { - fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { - self.peerset.discovered(peer_ids.into_iter().map(|peer_id| { - debug!(target: "sub-libp2p", "PSM <= Discovered({:?})", peer_id); - peer_id - })); + match self.peers.get_mut(&incoming.peer_id) { + Some(state @ PeerState::Incoming) => { + debug!(target: "sub-libp2p", "PSM => Reject({:?}, {:?}): Rejecting connections.", + index, incoming.peer_id); + debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: incoming.peer_id, + handler: NotifyHandler::All, + event: NotifsHandlerIn::Disable, + }); + *state = PeerState::Disabled { + open: SmallVec::new(), + banned_until: None + }; + } + peer => error!(target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", + peer) + } } } @@ -715,33 +798,43 @@ impl NetworkBehaviour for GenericProto { type OutEvent = GenericProtoOut; fn new_handler(&mut self) -> Self::ProtocolsHandler { - NotifsHandlerProto::new(self.legacy_protocol.clone(), self.notif_protocols.clone()) + NotifsHandlerProto::new( + self.legacy_protocol.clone(), + self.notif_protocols.clone(), + self.queue_size_report.clone() + ) } fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { Vec::new() } - fn inject_connected(&mut self, peer_id: PeerId, connected_point: ConnectedPoint) { - match (self.peers.entry(peer_id.clone()).or_insert(PeerState::Poisoned), connected_point) { - (st @ &mut PeerState::Requested, connected_point) | - (st @ &mut PeerState::PendingRequest { .. }, connected_point) => { - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Connection \ - requested by PSM (through {:?})", peer_id, connected_point + fn inject_connected(&mut self, _: &PeerId) { + } + + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + debug!(target: "sub-libp2p", "Libp2p => Connection ({:?},{:?}) to {} established.", + conn, endpoint, peer_id); + match (self.peers.entry(peer_id.clone()).or_insert(PeerState::Poisoned), endpoint) { + (st @ &mut PeerState::Requested, endpoint) | + (st @ &mut PeerState::PendingRequest { .. }, endpoint) => { + debug!(target: "sub-libp2p", + "Libp2p => Connected({}, {:?}): Connection was requested by PSM.", + peer_id, endpoint ); - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + *st = PeerState::Enabled { open: SmallVec::new() }; + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), - event: NotifsHandlerIn::Enable, + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Enable }); - *st = PeerState::Enabled { open: false, connected_point }; } - // Note: it may seem weird that "Banned" nodes get treated as if there were absent. + // Note: it may seem weird that "Banned" peers get treated as if they were absent. // This is because the word "Banned" means "temporarily prevent outgoing connections to - // this node", and not "banned" in the sense that we would refuse the node altogether. - (st @ &mut PeerState::Poisoned, connected_point @ ConnectedPoint::Listener { .. }) | - (st @ &mut PeerState::Banned { .. }, connected_point @ ConnectedPoint::Listener { .. }) => { + // this peer", and not "banned" in the sense that we would refuse the peer altogether. + (st @ &mut PeerState::Poisoned, endpoint @ ConnectedPoint::Listener { .. }) | + (st @ &mut PeerState::Banned { .. }, endpoint @ ConnectedPoint::Listener { .. }) => { let incoming_id = self.next_incoming_index.clone(); self.next_incoming_index.0 = match self.next_incoming_index.0.checked_add(1) { Some(v) => v, @@ -750,61 +843,79 @@ impl NetworkBehaviour for GenericProto { return } }; - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Incoming connection", - peer_id); - debug!(target: "sub-libp2p", "PSM <= Incoming({:?}, {:?}): Through {:?}", - incoming_id, peer_id, connected_point); + debug!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}): Incoming connection", + peer_id, endpoint); + debug!(target: "sub-libp2p", "PSM <= Incoming({}, {:?}).", + peer_id, incoming_id); self.peerset.incoming(peer_id.clone(), incoming_id); self.incoming.push(IncomingPeer { peer_id: peer_id.clone(), alive: true, incoming_id, }); - *st = PeerState::Incoming { connected_point }; + *st = PeerState::Incoming { }; } - (st @ &mut PeerState::Poisoned, connected_point) | - (st @ &mut PeerState::Banned { .. }, connected_point) => { + (st @ &mut PeerState::Poisoned, endpoint) | + (st @ &mut PeerState::Banned { .. }, endpoint) => { let banned_until = if let PeerState::Banned { until } = st { Some(*until) } else { None }; - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Requested by something \ - else than PSM, disabling", peer_id); - debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + debug!(target: "sub-libp2p", + "Libp2p => Connected({},{:?}): Not requested by PSM, disabling.", + peer_id, endpoint); + *st = PeerState::Disabled { open: SmallVec::new(), banned_until }; + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), - event: NotifsHandlerIn::Disable, + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Disable }); - *st = PeerState::Disabled { open: false, connected_point, banned_until }; } - st => { - // This is a serious bug either in this state machine or in libp2p. - error!(target: "sub-libp2p", "Received inject_connected for \ - already-connected node; state is {:?}", st - ); + (PeerState::Incoming { .. }, _) => { + debug!(target: "sub-libp2p", + "Secondary connection {:?} to {} waiting for PSM decision.", + conn, peer_id); + }, + + (PeerState::Enabled { .. }, _) => { + debug!(target: "sub-libp2p", "Handler({},{:?}) <= Enable secondary connection", + peer_id, conn); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: peer_id.clone(), + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Enable + }); + } + + (PeerState::Disabled { .. }, _) | (PeerState::DisabledPendingEnable { .. }, _) => { + debug!(target: "sub-libp2p", "Handler({},{:?}) <= Disable secondary connection", + peer_id, conn); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: peer_id.clone(), + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Disable + }); } } } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - match self.peers.remove(peer_id) { - None | Some(PeerState::Requested) | Some(PeerState::PendingRequest { .. }) | - Some(PeerState::Banned { .. }) => - // This is a serious bug either in this state machine or in libp2p. - error!(target: "sub-libp2p", "Received inject_disconnected for non-connected \ - node {:?}", peer_id), - - Some(PeerState::Disabled { open, banned_until, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was disabled \ - (through {:?})", peer_id, endpoint); - if let Some(until) = banned_until { - self.peers.insert(peer_id.clone(), PeerState::Banned { until }); - } - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + debug!(target: "sub-libp2p", "Libp2p => Connection ({:?},{:?}) to {} closed.", + conn, endpoint, peer_id); + match self.peers.get_mut(peer_id) { + Some(PeerState::Disabled { open, .. }) | + Some(PeerState::DisabledPendingEnable { open, .. }) | + Some(PeerState::Enabled { open, .. }) => { + // Check if the "link" to the peer is already considered closed, + // i.e. there is no connection that is open for custom protocols, + // in which case `CustomProtocolClosed` was already emitted. + let closed = open.is_empty(); + open.retain(|c| c != conn); + if !closed { + debug!(target: "sub-libp2p", "External API <= Closed({})", peer_id); let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), @@ -813,52 +924,52 @@ impl NetworkBehaviour for GenericProto { self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } } + _ => {} + } + } - Some(PeerState::DisabledPendingEnable { open, timer_deadline, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was disabled \ - (through {:?}) but pending enable", peer_id, endpoint); - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); - self.peerset.dropped(peer_id.clone()); - self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = GenericProtoOut::CustomProtocolClosed { - peer_id: peer_id.clone(), - reason: "Disconnected by libp2p".into(), - }; + fn inject_disconnected(&mut self, peer_id: &PeerId) { + match self.peers.remove(peer_id) { + None | Some(PeerState::Requested) | Some(PeerState::PendingRequest { .. }) | + Some(PeerState::Banned { .. }) => + // This is a serious bug either in this state machine or in libp2p. + error!(target: "sub-libp2p", + "`inject_disconnected` called for unknown peer {}", + peer_id), - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + Some(PeerState::Disabled { banned_until, .. }) => { + debug!(target: "sub-libp2p", "Libp2p => Disconnected({}): Was disabled.", peer_id); + if let Some(until) = banned_until { + self.peers.insert(peer_id.clone(), PeerState::Banned { until }); } } - Some(PeerState::Enabled { open, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was enabled \ - (through {:?})", peer_id, endpoint); - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); + Some(PeerState::DisabledPendingEnable { timer_deadline, .. }) => { + debug!(target: "sub-libp2p", + "Libp2p => Disconnected({}): Was disabled but pending enable.", + peer_id); + debug!(target: "sub-libp2p", "PSM <= Dropped({})", peer_id); self.peerset.dropped(peer_id.clone()); + self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); + } + Some(PeerState::Enabled { .. }) => { + debug!(target: "sub-libp2p", "Libp2p => Disconnected({}): Was enabled.", peer_id); + debug!(target: "sub-libp2p", "PSM <= Dropped({})", peer_id); + self.peerset.dropped(peer_id.clone()); let ban_dur = Uniform::new(5, 10).sample(&mut rand::thread_rng()); self.peers.insert(peer_id.clone(), PeerState::Banned { until: Instant::now() + Duration::from_secs(ban_dur) }); - - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = GenericProtoOut::CustomProtocolClosed { - peer_id: peer_id.clone(), - reason: "Disconnected by libp2p".into(), - }; - - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); - } } // In the incoming state, we don't report "Dropped". Instead we will just ignore the // corresponding Accept/Reject. - Some(PeerState::Incoming { .. }) => { + Some(PeerState::Incoming { }) => { if let Some(state) = self.incoming.iter_mut().find(|i| i.peer_id == *peer_id) { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was in incoming \ - mode (id {:?}, through {:?})", peer_id, state.incoming_id, endpoint); + debug!(target: "sub-libp2p", + "Libp2p => Disconnected({}): Was in incoming mode with id {:?}.", + peer_id, state.incoming_id); state.alive = false; } else { error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in incoming \ @@ -867,7 +978,7 @@ impl NetworkBehaviour for GenericProto { } Some(PeerState::Poisoned) => - error!(target: "sub-libp2p", "State of {:?} is poisoned", peer_id), + error!(target: "sub-libp2p", "State of peer {} is poisoned", peer_id), } } @@ -878,13 +989,13 @@ impl NetworkBehaviour for GenericProto { fn inject_dial_failure(&mut self, peer_id: &PeerId) { if let Entry::Occupied(mut entry) = self.peers.entry(peer_id.clone()) { match mem::replace(entry.get_mut(), PeerState::Poisoned) { - // The node is not in our list. + // The peer is not in our list. st @ PeerState::Banned { .. } => { trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); *entry.into_mut() = st; }, - // "Basic" situation: we failed to reach a node that the peerset requested. + // "Basic" situation: we failed to reach a peer that the peerset requested. PeerState::Requested | PeerState::PendingRequest { .. } => { debug!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); *entry.into_mut() = PeerState::Banned { @@ -894,7 +1005,7 @@ impl NetworkBehaviour for GenericProto { self.peerset.dropped(peer_id.clone()) }, - // We can still get dial failures even if we are already connected to the node, + // We can still get dial failures even if we are already connected to the peer, // as an extra diagnostic for an earlier attempt. st @ PeerState::Disabled { .. } | st @ PeerState::Enabled { .. } | st @ PeerState::DisabledPendingEnable { .. } | st @ PeerState::Incoming { .. } => { @@ -907,99 +1018,137 @@ impl NetworkBehaviour for GenericProto { } } else { - // The node is not in our list. + // The peer is not in our list. trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); } } - fn inject_node_event( + fn inject_event( &mut self, source: PeerId, + connection: ConnectionId, event: NotifsHandlerOut, ) { match event { - NotifsHandlerOut::Closed { reason } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Closed: {}", source, reason); + NotifsHandlerOut::Closed { endpoint, reason } => { + debug!(target: "sub-libp2p", + "Handler({:?}) => Endpoint {:?} closed for custom protocols: {}", + source, endpoint, reason); let mut entry = if let Entry::Occupied(entry) = self.peers.entry(source.clone()) { entry } else { - error!(target: "sub-libp2p", "State mismatch in the custom protos handler"); + error!(target: "sub-libp2p", "Closed: State mismatch in the custom protos handler"); return }; - debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); - let event = GenericProtoOut::CustomProtocolClosed { - reason, - peer_id: source.clone(), - }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); - - match mem::replace(entry.get_mut(), PeerState::Poisoned) { - PeerState::Enabled { open, connected_point } => { - debug_assert!(open); - - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", source); - self.peerset.dropped(source.clone()); + let last = match mem::replace(entry.get_mut(), PeerState::Poisoned) { + PeerState::Enabled { mut open } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", source); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: source.clone(), + handler: NotifyHandler::One(connection), event: NotifsHandlerIn::Disable, }); + let last = open.is_empty(); + + if last { + debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", source); + self.peerset.dropped(source.clone()); + *entry.into_mut() = PeerState::Disabled { + open, + banned_until: None + }; + } else { + *entry.into_mut() = PeerState::Enabled { open }; + } + + last + }, + PeerState::Disabled { mut open, banned_until } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); + let last = open.is_empty(); *entry.into_mut() = PeerState::Disabled { - open: false, - connected_point, - banned_until: None + open, + banned_until }; + last }, - PeerState::Disabled { open, connected_point, banned_until } => { - debug_assert!(open); - *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until }; - }, - PeerState::DisabledPendingEnable { open, connected_point, timer, timer_deadline } => { - debug_assert!(open); + PeerState::DisabledPendingEnable { + mut open, + timer, + timer_deadline + } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); + let last = open.is_empty(); *entry.into_mut() = PeerState::DisabledPendingEnable { - open: false, - connected_point, + open, timer, timer_deadline }; + last }, - _ => error!(target: "sub-libp2p", "State mismatch in the custom protos handler"), + state => { + error!(target: "sub-libp2p", + "Unexpected state in the custom protos handler: {:?}", + state); + return + } + }; + + if last { + debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); + let event = GenericProtoOut::CustomProtocolClosed { + reason, + peer_id: source.clone(), + }; + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } else { + debug!(target: "sub-libp2p", "Secondary connection closed custom protocol."); } } - NotifsHandlerOut::Open => { - debug!(target: "sub-libp2p", "Handler({:?}) => Open", source); - let endpoint = match self.peers.get_mut(&source) { - Some(PeerState::Enabled { ref mut open, ref connected_point }) | - Some(PeerState::DisabledPendingEnable { ref mut open, ref connected_point, .. }) | - Some(PeerState::Disabled { ref mut open, ref connected_point, .. }) if !*open => { - *open = true; - connected_point.clone() + NotifsHandlerOut::Open { endpoint } => { + debug!(target: "sub-libp2p", + "Handler({:?}) => Endpoint {:?} open for custom protocols.", + source, endpoint); + + let first = match self.peers.get_mut(&source) { + Some(PeerState::Enabled { ref mut open, .. }) | + Some(PeerState::DisabledPendingEnable { ref mut open, .. }) | + Some(PeerState::Disabled { ref mut open, .. }) => { + let first = open.is_empty(); + open.push(connection); + first } - _ => { - error!(target: "sub-libp2p", "State mismatch in the custom protos handler"); + state => { + error!(target: "sub-libp2p", + "Open: Unexpected state in the custom protos handler: {:?}", + state); return } }; - debug!(target: "sub-libp2p", "External API <= Open({:?})", source); - let event = GenericProtoOut::CustomProtocolOpen { - peer_id: source, - endpoint, - }; - - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + if first { + debug!(target: "sub-libp2p", "External API <= Open({:?})", source); + let event = GenericProtoOut::CustomProtocolOpen { peer_id: source }; + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } else { + debug!(target: "sub-libp2p", "Secondary connection opened custom protocol."); + } } NotifsHandlerOut::CustomMessage { message } => { debug_assert!(self.is_open(&source)); trace!(target: "sub-libp2p", "Handler({:?}) => Message", source); trace!(target: "sub-libp2p", "External API <= Message({:?})", source); - let event = GenericProtoOut::CustomMessage { + let event = GenericProtoOut::LegacyMessage { peer_id: source, message, }; @@ -1007,7 +1156,7 @@ impl NetworkBehaviour for GenericProto { self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } - NotifsHandlerOut::Notification { protocol_name, engine_id, message } => { + NotifsHandlerOut::Notification { protocol_name, message } => { debug_assert!(self.is_open(&source)); trace!( target: "sub-libp2p", @@ -1015,18 +1164,11 @@ impl NetworkBehaviour for GenericProto { source, str::from_utf8(&protocol_name) ); - trace!(target: "sub-libp2p", "External API <= Message({:?})", source); - let event = GenericProtoOut::CustomMessage { + trace!(target: "sub-libp2p", "External API <= Message({:?}, {:?})", protocol_name, source); + let event = GenericProtoOut::Notification { peer_id: source, - message: { - let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { - engine_id, - data: message.to_vec(), - }); - - // Note that we clone `message` here. - From::from(&message.encode()[..]) - }, + protocol_name, + message, }; self.events.push(NetworkBehaviourAction::GenerateEvent(event)); @@ -1051,11 +1193,12 @@ impl NetworkBehaviour for GenericProto { } NotifsHandlerOut::ProtocolError { error, .. } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Severe protocol error: {:?}", + warn!(target: "sub-libp2p", + "Handler({:?}) => Severe protocol error: {:?}", source, error); - // A severe protocol error happens when we detect a "bad" node, such as a node on - // a different chain, or a node that doesn't speak the same protocol(s). We - // decrease the node's reputation, hence lowering the chances we try this node + // A severe protocol error happens when we detect a "bad" peer, such as a per on + // a different chain, or a peer that doesn't speak the same protocol(s). We + // decrease the peer's reputation, hence lowering the chances we try this peer // again in the short term. self.peerset.report_peer( source.clone(), @@ -1109,27 +1252,34 @@ impl NetworkBehaviour for GenericProto { } debug!(target: "sub-libp2p", "Libp2p <= Dial {:?} now that ban has expired", peer_id); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: peer_id.clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: peer_id.clone(), + condition: DialPeerCondition::Disconnected + }); *peer_state = PeerState::Requested; } - PeerState::DisabledPendingEnable { mut timer, connected_point, open, timer_deadline } => { + PeerState::DisabledPendingEnable { + mut timer, + open, + timer_deadline + } => { if let Poll::Pending = Pin::new(&mut timer).poll(cx) { *peer_state = PeerState::DisabledPendingEnable { timer, - connected_point, open, timer_deadline }; continue; } - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable now that ban has expired", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + debug!(target: "sub-libp2p", "Handler({:?}) <= Enable (ban expired)", peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *peer_state = PeerState::Enabled { connected_point, open }; + *peer_state = PeerState::Enabled { open }; } st @ _ => *peer_state = st, diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs index d6d9919d3e14df0c2261731cddb0811aa291f920..7e597f1be69f9d51b8f7982fc5bf1d4b38b9c765 100644 --- a/client/network/src/protocol/generic_proto/handler/group.rs +++ b/client/network/src/protocol/generic_proto/handler/group.rs @@ -51,10 +51,8 @@ use crate::protocol::generic_proto::{ handler::notif_out::{NotifsOutHandlerProto, NotifsOutHandler, NotifsOutHandlerIn, NotifsOutHandlerOut}, upgrade::{NotificationsIn, NotificationsOut, NotificationsHandshakeError, RegisteredProtocol, UpgradeCollec}, }; -use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; use bytes::BytesMut; -use codec::Encode as _; use libp2p::core::{either::{EitherError, EitherOutput}, ConnectedPoint, PeerId}; use libp2p::core::upgrade::{EitherUpgrade, UpgradeError, SelectUpgrade, InboundUpgrade, OutboundUpgrade}; use libp2p::swarm::{ @@ -65,9 +63,9 @@ use libp2p::swarm::{ SubstreamProtocol, NegotiatedSubstream, }; -use log::error; -use sp_runtime::ConsensusEngineId; -use std::{borrow::Cow, error, io, task::{Context, Poll}}; +use log::{debug, error}; +use prometheus_endpoint::HistogramVec; +use std::{borrow::Cow, error, io, str, task::{Context, Poll}}; /// Implements the `IntoProtocolsHandler` trait of libp2p. /// @@ -78,10 +76,10 @@ use std::{borrow::Cow, error, io, task::{Context, Poll}}; /// See the documentation at the module level for more information. pub struct NotifsHandlerProto { /// Prototypes for handlers for inbound substreams. - in_handlers: Vec<(NotifsInHandlerProto, ConsensusEngineId)>, + in_handlers: Vec, /// Prototypes for handlers for outbound substreams. - out_handlers: Vec<(NotifsOutHandlerProto, ConsensusEngineId)>, + out_handlers: Vec, /// Prototype for handler for backwards-compatibility. legacy: LegacyProtoHandlerProto, @@ -92,10 +90,10 @@ pub struct NotifsHandlerProto { /// See the documentation at the module level for more information. pub struct NotifsHandler { /// Handlers for inbound substreams. - in_handlers: Vec<(NotifsInHandler, ConsensusEngineId)>, + in_handlers: Vec, /// Handlers for outbound substreams. - out_handlers: Vec<(NotifsOutHandler, ConsensusEngineId)>, + out_handlers: Vec, /// Handler for backwards-compatibility. legacy: LegacyProtoHandler, @@ -121,7 +119,7 @@ impl IntoProtocolsHandler for NotifsHandlerProto { fn inbound_protocol(&self) -> SelectUpgrade, RegisteredProtocol> { let in_handlers = self.in_handlers.iter() - .map(|(h, _)| h.inbound_protocol()) + .map(|h| h.inbound_protocol()) .collect::>(); SelectUpgrade::new(in_handlers, self.legacy.inbound_protocol()) @@ -131,11 +129,11 @@ impl IntoProtocolsHandler for NotifsHandlerProto { NotifsHandler { in_handlers: self.in_handlers .into_iter() - .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .map(|p| p.into_handler(remote_peer_id, connected_point)) .collect(), out_handlers: self.out_handlers .into_iter() - .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .map(|p| p.into_handler(remote_peer_id, connected_point)) .collect(), legacy: self.legacy.into_handler(remote_peer_id, connected_point), enabled: EnabledState::Initial, @@ -145,7 +143,7 @@ impl IntoProtocolsHandler for NotifsHandlerProto { } /// Event that can be received by a `NotifsHandler`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum NotifsHandlerIn { /// The node should start using custom protocols. Enable, @@ -155,7 +153,8 @@ pub enum NotifsHandlerIn { /// Sends a message through the custom protocol substream. /// - /// > **Note**: This must **not** be an encoded `ConsensusMessage` message. + /// > **Note**: This must **not** be a `ConsensusMessage`, `Transactions`, or + /// > `BlockAnnounce` message. SendLegacy { /// The message to send. message: Vec, @@ -166,17 +165,13 @@ pub enum NotifsHandlerIn { /// Name of the protocol for the message. /// /// Must match one of the registered protocols. For backwards-compatibility reasons, if - /// the remote doesn't support this protocol, we use the legacy substream to send a - /// `ConsensusMessage` message. + /// the remote doesn't support this protocol, we use the legacy substream. protocol_name: Cow<'static, [u8]>, - /// The engine ID to use, in case we need to send this message over the legacy substream. + /// Message to send on the legacy substream if the protocol isn't available. /// - /// > **Note**: Ideally this field wouldn't be necessary, and we would deduce the engine - /// > ID from the existing handlers. However, it is possible (especially in test - /// > situations) that we open connections before all the notification protocols - /// > have been registered, in which case we always rely on the legacy substream. - engine_id: ConsensusEngineId, + /// This corresponds to what you would have sent with `SendLegacy`. + encoded_fallback_message: Vec, /// The message to send. message: Vec, @@ -186,13 +181,18 @@ pub enum NotifsHandlerIn { /// Event that can be emitted by a `NotifsHandler`. #[derive(Debug)] pub enum NotifsHandlerOut { - /// Opened the substreams with the remote. - Open, + /// The connection is open for custom protocols. + Open { + /// The endpoint of the connection that is open for custom protocols. + endpoint: ConnectedPoint, + }, - /// Closed the substreams with the remote. + /// The connection is closed for custom protocols. Closed { - /// Reason why the substream closed, for diagnostic purposes. + /// The reason for closing, for diagnostic purposes. reason: Cow<'static, str>, + /// The endpoint of the connection that closed for custom protocols. + endpoint: ConnectedPoint, }, /// Received a non-gossiping message on the legacy substream. @@ -206,17 +206,10 @@ pub enum NotifsHandlerOut { /// Received a message on a custom protocol substream. Notification { - /// Engine corresponding to the message. + /// Name of the protocol of the message. protocol_name: Cow<'static, [u8]>, - /// For legacy reasons, the name to use if we had received the message from the legacy - /// substream. - engine_id: ConsensusEngineId, - /// Message that has been received. - /// - /// If `protocol_name` is `None`, this decodes to a `Message`. If `protocol_name` is `Some`, - /// this is directly a gossiping message. message: BytesMut, }, @@ -238,12 +231,30 @@ pub enum NotifsHandlerOut { impl NotifsHandlerProto { /// Builds a new handler. - pub fn new(legacy: RegisteredProtocol, list: impl Into, ConsensusEngineId, Vec)>>) -> Self { + /// + /// The `queue_size_report` is an optional Prometheus metric that can report the size of the + /// messages queue. If passed, it must have one label for the protocol name. + pub fn new(legacy: RegisteredProtocol, list: impl Into, Vec)>>, queue_size_report: Option) -> Self { let list = list.into(); + let out_handlers = list + .clone() + .into_iter() + .map(|(p, _)| { + let queue_size_report = queue_size_report.as_ref().and_then(|qs| { + if let Ok(utf8) = str::from_utf8(&p) { + Some(qs.with_label_values(&[utf8])) + } else { + log::warn!("Ignoring Prometheus metric because {:?} isn't UTF-8", p); + None + } + }); + NotifsOutHandlerProto::new(p, queue_size_report) + }).collect(); + NotifsHandlerProto { - in_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsInHandlerProto::new(p), e)).collect(), - out_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsOutHandlerProto::new(p), e)).collect(), + in_handlers: list.clone().into_iter().map(|(p, _)| NotifsInHandlerProto::new(p)).collect(), + out_handlers, legacy: LegacyProtoHandlerProto::new(legacy), } } @@ -266,7 +277,7 @@ impl ProtocolsHandler for NotifsHandler { fn listen_protocol(&self) -> SubstreamProtocol { let in_handlers = self.in_handlers.iter() - .map(|h| h.0.listen_protocol().into_upgrade().1) + .map(|h| h.listen_protocol().into_upgrade().1) .collect::>(); let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1); @@ -279,7 +290,7 @@ impl ProtocolsHandler for NotifsHandler { ) { match out { EitherOutput::First((out, num)) => - self.in_handlers[num].0.inject_fully_negotiated_inbound(out), + self.in_handlers[num].inject_fully_negotiated_inbound(out), EitherOutput::Second(out) => self.legacy.inject_fully_negotiated_inbound(out), } @@ -292,7 +303,7 @@ impl ProtocolsHandler for NotifsHandler { ) { match (out, num) { (EitherOutput::First(out), Some(num)) => - self.out_handlers[num].0.inject_fully_negotiated_outbound(out, ()), + self.out_handlers[num].inject_fully_negotiated_outbound(out, ()), (EitherOutput::Second(out), None) => self.legacy.inject_fully_negotiated_outbound(out, ()), _ => error!("inject_fully_negotiated_outbound called with wrong parameters"), @@ -302,54 +313,53 @@ impl ProtocolsHandler for NotifsHandler { fn inject_event(&mut self, message: NotifsHandlerIn) { match message { NotifsHandlerIn::Enable => { + if let EnabledState::Enabled = self.enabled { + debug!("enabling already-enabled handler"); + } self.enabled = EnabledState::Enabled; self.legacy.inject_event(LegacyProtoHandlerIn::Enable); - for (handler, _) in &mut self.out_handlers { + for handler in &mut self.out_handlers { handler.inject_event(NotifsOutHandlerIn::Enable { initial_message: vec![] }); } for num in self.pending_in.drain(..) { - self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Accept(vec![])); + self.in_handlers[num].inject_event(NotifsInHandlerIn::Accept(vec![])); } }, NotifsHandlerIn::Disable => { + if let EnabledState::Disabled = self.enabled { + debug!("disabling already-disabled handler"); + } self.legacy.inject_event(LegacyProtoHandlerIn::Disable); // The notifications protocols start in the disabled state. If we were in the // "Initial" state, then we shouldn't disable the notifications protocols again. if self.enabled != EnabledState::Initial { - for (handler, _) in &mut self.out_handlers { + for handler in &mut self.out_handlers { handler.inject_event(NotifsOutHandlerIn::Disable); } } self.enabled = EnabledState::Disabled; for num in self.pending_in.drain(..) { - self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Refuse); + self.in_handlers[num].inject_event(NotifsInHandlerIn::Refuse); } }, NotifsHandlerIn::SendLegacy { message } => self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { message }), - NotifsHandlerIn::SendNotification { message, engine_id, protocol_name } => { - for (handler, ngn_id) in &mut self.out_handlers { + NotifsHandlerIn::SendNotification { message, encoded_fallback_message, protocol_name } => { + for handler in &mut self.out_handlers { if handler.protocol_name() != &protocol_name[..] { - break; + continue; } if handler.is_open() { handler.inject_event(NotifsOutHandlerIn::Send(message)); return; - } else { - debug_assert_eq!(engine_id, *ngn_id); } } - let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { - engine_id, - data: message, - }); - self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { - message: message.encode() + message: encoded_fallback_message, }); }, } @@ -362,21 +372,21 @@ impl ProtocolsHandler for NotifsHandler { ) { match (err, num) { (ProtocolsHandlerUpgrErr::Timeout, Some(num)) => - self.out_handlers[num].0.inject_dial_upgrade_error( + self.out_handlers[num].inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Timeout ), (ProtocolsHandlerUpgrErr::Timeout, None) => self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timeout), (ProtocolsHandlerUpgrErr::Timer, Some(num)) => - self.out_handlers[num].0.inject_dial_upgrade_error( + self.out_handlers[num].inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Timer ), (ProtocolsHandlerUpgrErr::Timer, None) => self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timer), (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), Some(num)) => - self.out_handlers[num].0.inject_dial_upgrade_error( + self.out_handlers[num].inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) ), @@ -386,7 +396,7 @@ impl ProtocolsHandler for NotifsHandler { ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) ), (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::A(err))), Some(num)) => - self.out_handlers[num].0.inject_dial_upgrade_error( + self.out_handlers[num].inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) ), @@ -407,7 +417,7 @@ impl ProtocolsHandler for NotifsHandler { return KeepAlive::Yes; } - for (handler, _) in &self.in_handlers { + for handler in &self.in_handlers { let val = handler.connection_keep_alive(); if val.is_yes() { return KeepAlive::Yes; @@ -415,7 +425,7 @@ impl ProtocolsHandler for NotifsHandler { if ret < val { ret = val; } } - for (handler, _) in &self.out_handlers { + for handler in &self.out_handlers { let val = handler.connection_keep_alive(); if val.is_yes() { return KeepAlive::Yes; @@ -432,7 +442,7 @@ impl ProtocolsHandler for NotifsHandler { ) -> Poll< ProtocolsHandlerEvent > { - for (handler_num, (handler, engine_id)) in self.in_handlers.iter_mut().enumerate() { + for (handler_num, handler) in self.in_handlers.iter_mut().enumerate() { while let Poll::Ready(ev) = handler.poll(cx) { match ev { ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } => @@ -453,7 +463,6 @@ impl ProtocolsHandler for NotifsHandler { if self.legacy.is_open() { let msg = NotifsHandlerOut::Notification { message, - engine_id: *engine_id, protocol_name: handler.protocol_name().to_owned().into(), }; return Poll::Ready(ProtocolsHandlerEvent::Custom(msg)); @@ -463,7 +472,7 @@ impl ProtocolsHandler for NotifsHandler { } } - for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() { + for (handler_num, handler) in self.out_handlers.iter_mut().enumerate() { while let Poll::Ready(ev) = handler.poll(cx) { match ev { ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => @@ -493,13 +502,13 @@ impl ProtocolsHandler for NotifsHandler { protocol: protocol.map_upgrade(EitherUpgrade::B), info: None, }), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { .. }) => + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { endpoint, .. }) => return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::Open + NotifsHandlerOut::Open { endpoint } )), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason }) => + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { endpoint, reason }) => return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::Closed { reason } + NotifsHandlerOut::Closed { endpoint, reason } )), ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => return Poll::Ready(ProtocolsHandlerEvent::Custom( diff --git a/client/network/src/protocol/generic_proto/handler/legacy.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs index a2d2fc9246d1c79b761d5e5eb1ce375fba33b49a..bc84fd847c9c4ba44dbffcf5a7735c18c39f7776 100644 --- a/client/network/src/protocol/generic_proto/handler/legacy.rs +++ b/client/network/src/protocol/generic_proto/handler/legacy.rs @@ -40,9 +40,8 @@ use std::{pin::Pin, task::{Context, Poll}}; /// it is turned into a `LegacyProtoHandler`. It then handles all communications that are specific /// to Substrate on that single connection. /// -/// Note that there can be multiple instance of this struct simultaneously for same peer. However -/// if that happens, only one main instance can communicate with the outer layers of the code. In -/// other words, the outer layers of the code only ever see one handler. +/// Note that there can be multiple instance of this struct simultaneously for same peer, +/// if there are multiple established connections to the peer. /// /// ## State of the handler /// @@ -61,6 +60,7 @@ use std::{pin::Pin, task::{Context, Poll}}; /// these states. For example, if the handler reports a network misbehaviour, it will close the /// substreams but it is the role of the user to send a `Disabled` event if it wants the connection /// to close. Otherwise, the handler will try to reopen substreams. +/// /// The handler starts in the "Initializing" state and must be transitionned to Enabled or Disabled /// as soon as possible. /// @@ -111,7 +111,7 @@ impl IntoProtocolsHandler for LegacyProtoHandlerProto { fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { LegacyProtoHandler { protocol: self.protocol, - endpoint: connected_point.to_endpoint(), + endpoint: connected_point.clone(), remote_peer_id: remote_peer_id.clone(), state: ProtocolState::Init { substreams: SmallVec::new(), @@ -136,7 +136,7 @@ pub struct LegacyProtoHandler { /// Whether we are the connection dialer or listener. Used to determine who, between the local /// node and the remote node, has priority. - endpoint: Endpoint, + endpoint: ConnectedPoint, /// Queue of events to send to the outside. /// @@ -218,12 +218,16 @@ pub enum LegacyProtoHandlerOut { CustomProtocolOpen { /// Version of the protocol that has been opened. version: u8, + /// The connected endpoint. + endpoint: ConnectedPoint, }, /// Closed a custom protocol with the remote. CustomProtocolClosed { /// Reason why the substream closed, for diagnostic purposes. reason: Cow<'static, str>, + /// The connected endpoint. + endpoint: ConnectedPoint, }, /// Receives a message on a custom protocol substream. @@ -272,7 +276,7 @@ impl LegacyProtoHandler { ProtocolState::Init { substreams: incoming, .. } => { if incoming.is_empty() { - if let Endpoint::Dialer = self.endpoint { + if let ConnectedPoint::Dialer { .. } = self.endpoint { self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(self.protocol.clone()), info: (), @@ -281,10 +285,10 @@ impl LegacyProtoHandler { ProtocolState::Opening { deadline: Delay::new(Duration::from_secs(60)) } - } else { let event = LegacyProtoHandlerOut::CustomProtocolOpen { - version: incoming[0].protocol_version() + version: incoming[0].protocol_version(), + endpoint: self.endpoint.clone() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { @@ -404,6 +408,7 @@ impl LegacyProtoHandler { if substreams.is_empty() { let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: "All substreams have been closed by the remote".into(), + endpoint: self.endpoint.clone() }; self.state = ProtocolState::Disabled { shutdown: shutdown.into_iter().collect(), @@ -416,6 +421,7 @@ impl LegacyProtoHandler { if substreams.is_empty() { let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: format!("Error on the last substream: {:?}", err).into(), + endpoint: self.endpoint.clone() }; self.state = ProtocolState::Disabled { shutdown: shutdown.into_iter().collect(), @@ -479,7 +485,8 @@ impl LegacyProtoHandler { ProtocolState::Opening { .. } => { let event = LegacyProtoHandlerOut::CustomProtocolOpen { - version: substream.protocol_version() + version: substream.protocol_version(), + endpoint: self.endpoint.clone() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs index 4e16fb1af419f4aebb2748b09c477119f1165309..83923154bd6256de3bb4e983796807093f7019ad 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_in.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -36,7 +36,7 @@ use libp2p::swarm::{ }; use log::{error, warn}; use smallvec::SmallVec; -use std::{borrow::Cow, fmt, pin::Pin, str, task::{Context, Poll}}; +use std::{borrow::Cow, fmt, pin::Pin, task::{Context, Poll}}; /// Implements the `IntoProtocolsHandler` trait of libp2p. /// @@ -72,7 +72,7 @@ pub struct NotifsInHandler { } /// Event that can be received by a `NotifsInHandler`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum NotifsInHandlerIn { /// Can be sent back as a response to an `OpenRequest`. Contains the status message to send /// to the remote. @@ -156,16 +156,19 @@ impl ProtocolsHandler for NotifsInHandler { &mut self, (msg, proto): >::Output ) { + // If a substream already exists, we drop it and replace it with the new incoming one. if self.substream.is_some() { - warn!( - target: "sub-libp2p", - "Received duplicate inbound notifications substream for {:?}", - str::from_utf8(self.in_protocol.protocol_name()), - ); - return; + self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); } + // Note that we drop the existing substream, which will send an equivalent to a TCP "RST" + // to the remote and force-close the substream. It might seem like an unclean way to get + // rid of a substream. However, keep in mind that it is invalid for the remote to open + // multiple such substreams, and therefore sending a "RST" is the correct thing to do. + // Also note that we have already closed our writing side during the initial handshake, + // and we can't close "more" than that anyway. self.substream = Some(proto); + self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); self.pending_accept_refuses = self.pending_accept_refuses .checked_add(1) @@ -235,8 +238,15 @@ impl ProtocolsHandler for NotifsInHandler { match self.substream.as_mut().map(|s| Stream::poll_next(Pin::new(s), cx)) { None | Some(Poll::Pending) => {}, - Some(Poll::Ready(Some(Ok(msg)))) => - return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(msg))), + Some(Poll::Ready(Some(Ok(msg)))) => { + if self.pending_accept_refuses != 0 { + warn!( + target: "sub-libp2p", + "Bad state in inbound-only handler: notif before accepting substream" + ); + } + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(msg))) + }, Some(Poll::Ready(None)) | Some(Poll::Ready(Some(Err(_)))) => { self.substream = None; return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs index 8c64491d997171df73606c2852765c10c9f3f21b..b5d6cd61ada2a9e54bfa5873e0f7992ab1dfcfcb 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_out.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs @@ -33,7 +33,8 @@ use libp2p::swarm::{ SubstreamProtocol, NegotiatedSubstream, }; -use log::error; +use log::{debug, warn, error}; +use prometheus_endpoint::Histogram; use smallvec::SmallVec; use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; use wasm_timer::Instant; @@ -56,14 +57,17 @@ const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5); pub struct NotifsOutHandlerProto { /// Name of the protocol to negotiate. protocol_name: Cow<'static, [u8]>, + /// Optional Prometheus histogram to report message queue size variations. + queue_size_report: Option, } impl NotifsOutHandlerProto { /// Builds a new [`NotifsOutHandlerProto`]. Will use the given protocol name for the /// notifications substream. - pub fn new(protocol_name: impl Into>) -> Self { + pub fn new(protocol_name: impl Into>, queue_size_report: Option) -> Self { NotifsOutHandlerProto { protocol_name: protocol_name.into(), + queue_size_report, } } } @@ -75,12 +79,14 @@ impl IntoProtocolsHandler for NotifsOutHandlerProto { DeniedUpgrade } - fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler { + fn into_handler(self, peer_id: &PeerId, _: &ConnectedPoint) -> Self::Handler { NotifsOutHandler { protocol_name: self.protocol_name, when_connection_open: Instant::now(), + queue_size_report: self.queue_size_report, state: State::Disabled, events_queue: SmallVec::new(), + peer_id: peer_id.clone(), } } } @@ -103,11 +109,17 @@ pub struct NotifsOutHandler { /// When the connection with the remote has been successfully established. when_connection_open: Instant, + /// Optional prometheus histogram to report message queue sizes variations. + queue_size_report: Option, + /// Queue of events to send to the outside. /// /// This queue must only ever be modified to insert elements at the back, or remove the first /// element. events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + + /// Who we are connected to. + peer_id: PeerId, } /// Our relationship with the node we're connected to. @@ -244,8 +256,8 @@ impl ProtocolsHandler for NotifsOutHandler { // Any other situation should never happen. State::Disabled | State::Refused | State::Open { .. } | State::DisabledOpen(_) => - error!("State mismatch in notifications handler: substream already open"), - State::Poisoned => error!("Notifications handler in a poisoned state"), + error!("☎️ State mismatch in notifications handler: substream already open"), + State::Poisoned => error!("☎️ Notifications handler in a poisoned state"), } } @@ -268,9 +280,9 @@ impl ProtocolsHandler for NotifsOutHandler { // be recovered. When in doubt, let's drop the existing substream and // open a new one. if sub.close().now_or_never().is_none() { - log::warn!( + warn!( target: "sub-libp2p", - "Improperly closed outbound notifications substream" + "📞 Improperly closed outbound notifications substream" ); } @@ -281,37 +293,47 @@ impl ProtocolsHandler for NotifsOutHandler { }); self.state = State::Opening { initial_message }; }, - State::Opening { .. } | State::Refused | State::Open { .. } => - error!("Tried to enable notifications handler that was already enabled"), + st @ State::Opening { .. } | st @ State::Refused | st @ State::Open { .. } => { + debug!(target: "sub-libp2p", + "Tried to enable notifications handler that was already enabled"); + self.state = st; + } State::Poisoned => error!("Notifications handler in a poisoned state"), } } NotifsOutHandlerIn::Disable => { match mem::replace(&mut self.state, State::Poisoned) { - State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => - error!("Tried to disable notifications handler that was already disabled"), + st @ State::Disabled | st @ State::DisabledOpen(_) | st @ State::DisabledOpening => { + debug!(target: "sub-libp2p", + "Tried to disable notifications handler that was already disabled"); + self.state = st; + } State::Opening { .. } => self.state = State::DisabledOpening, State::Refused => self.state = State::Disabled, State::Open { substream, .. } => self.state = State::DisabledOpen(substream), - State::Poisoned => error!("Notifications handler in a poisoned state"), + State::Poisoned => error!("☎️ Notifications handler in a poisoned state"), } } NotifsOutHandlerIn::Send(msg) => if let State::Open { substream, .. } = &mut self.state { - if let Some(Ok(_)) = substream.send(msg).now_or_never() { - } else { - log::warn!( + if substream.push_message(msg).is_err() { + warn!( target: "sub-libp2p", - "Failed to push message to queue, dropped it" + "📞 Notifications queue with peer {} is full, dropped message (protocol: {:?})", + self.peer_id, + self.protocol_name, ); } + if let Some(metric) = &self.queue_size_report { + metric.observe(substream.queue_len() as f64); + } } else { // This is an API misuse. - log::warn!( + warn!( target: "sub-libp2p", - "Tried to send a notification on a disabled handler" + "📞 Tried to send a notification on a disabled handler" ); }, } @@ -321,14 +343,14 @@ impl ProtocolsHandler for NotifsOutHandler { match mem::replace(&mut self.state, State::Poisoned) { State::Disabled => {}, State::DisabledOpen(_) | State::Refused | State::Open { .. } => - error!("State mismatch in NotificationsOut"), + error!("☎️ State mismatch in NotificationsOut"), State::Opening { .. } => { self.state = State::Refused; let ev = NotifsOutHandlerOut::Refused; self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); }, State::DisabledOpening => self.state = State::Disabled, - State::Poisoned => error!("Notifications handler in a poisoned state"), + State::Poisoned => error!("☎️ Notifications handler in a poisoned state"), } } diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs index b63a725c074a5cc398d7707806d46226a3015f29..1bc6e745f887685af41a6c9cfdfc05da84af7918 100644 --- a/client/network/src/protocol/generic_proto/tests.rs +++ b/client/network/src/protocol/generic_proto/tests.rs @@ -18,7 +18,7 @@ use futures::{prelude::*, ready}; use codec::{Encode, Decode}; -use libp2p::core::nodes::listeners::ListenerId; +use libp2p::core::connection::{ConnectionId, ListenerId}; use libp2p::core::ConnectedPoint; use libp2p::swarm::{Swarm, ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction}; @@ -78,11 +78,11 @@ fn build_nodes() -> (Swarm, Swarm) { vec![] }, reserved_only: false, - reserved_nodes: Vec::new(), + priority_groups: Vec::new(), }); let behaviour = CustomProtoWithAddr { - inner: GenericProto::new(&b"test"[..], &[1], peerset), + inner: GenericProto::new(&b"test"[..], &[1], peerset, None), addrs: addrs .iter() .enumerate() @@ -148,20 +148,29 @@ impl NetworkBehaviour for CustomProtoWithAddr { list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.inner.inject_connected(peer_id, endpoint) + fn inject_connected(&mut self, peer_id: &PeerId) { + self.inner.inject_connected(peer_id) } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.inner.inject_disconnected(peer_id, endpoint) + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.inner.inject_disconnected(peer_id) } - fn inject_node_event( + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.inner.inject_connection_established(peer_id, conn, endpoint) + } + + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.inner.inject_connection_closed(peer_id, conn, endpoint) + } + + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent ) { - self.inner.inject_node_event(peer_id, event) + self.inner.inject_event(peer_id, connection, event) } fn poll( @@ -177,10 +186,6 @@ impl NetworkBehaviour for CustomProtoWithAddr { self.inner.poll(cx, params) } - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.inner.inject_replaced(peer_id, closed_endpoint, new_endpoint) - } - fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) { self.inner.inject_addr_reach_failure(peer_id, addr, error) } @@ -205,8 +210,8 @@ impl NetworkBehaviour for CustomProtoWithAddr { self.inner.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.inner.inject_listener_closed(id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.inner.inject_listener_closed(id, reason); } } @@ -245,7 +250,7 @@ fn two_nodes_transfer_lots_of_packets() { loop { match ready!(service2.poll_next_unpin(cx)) { Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, - Some(GenericProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::LegacyMessage { message, .. }) => { match Message::::decode(&mut &message[..]).unwrap() { Message::::BlockResponse(BlockResponse { id: _, blocks }) => { assert!(blocks.is_empty()); @@ -315,7 +320,7 @@ fn basic_two_nodes_requests_in_parallel() { loop { match ready!(service2.poll_next_unpin(cx)) { Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, - Some(GenericProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::LegacyMessage { message, .. }) => { let pos = to_receive.iter().position(|m| m.encode() == message).unwrap(); to_receive.remove(pos); if to_receive.is_empty() { diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs index 68898a08fe1af3c76e047d91b79acf0d112e8dde..f626110a3346cd414bab6a936b6f9fcd6290bd76 100644 --- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs +++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs @@ -38,13 +38,12 @@ use futures::{prelude::*, ready}; use futures_codec::Framed; use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade}; use log::error; -use std::{borrow::Cow, collections::VecDeque, io, iter, mem, pin::Pin, task::{Context, Poll}}; +use std::{borrow::Cow, collections::VecDeque, convert::TryFrom as _, io, iter, mem, pin::Pin, task::{Context, Poll}}; use unsigned_varint::codec::UviBytes; /// Maximum allowed size of the two handshake messages, in bytes. const MAX_HANDSHAKE_SIZE: usize = 1024; -/// Maximum number of buffered messages before we consider the remote unresponsive and kill the -/// substream. +/// Maximum number of buffered messages before we refuse to accept more. const MAX_PENDING_MESSAGES: usize = 256; /// Upgrade that accepts a substream, sends back a status message, then becomes a unidirectional @@ -164,12 +163,9 @@ where TSubstream: AsyncRead + AsyncWrite, { /// Sends the handshake in order to inform the remote that we accept the substream. pub fn send_handshake(&mut self, message: impl Into>) { - match self.handshake { - NotificationsInSubstreamHandshake::NotSent => {} - _ => { - error!(target: "sub-libp2p", "Tried to send handshake twice"); - return; - } + if !matches!(self.handshake, NotificationsInSubstreamHandshake::NotSent) { + error!(target: "sub-libp2p", "Tried to send handshake twice"); + return; } self.handshake = NotificationsInSubstreamHandshake::PendingSend(message.into()); @@ -189,8 +185,10 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin, match mem::replace(this.handshake, NotificationsInSubstreamHandshake::Sent) { NotificationsInSubstreamHandshake::Sent => return Stream::poll_next(this.socket.as_mut(), cx), - NotificationsInSubstreamHandshake::NotSent => - return Poll::Pending, + NotificationsInSubstreamHandshake::NotSent => { + *this.handshake = NotificationsInSubstreamHandshake::NotSent; + return Poll::Pending + }, NotificationsInSubstreamHandshake::PendingSend(msg) => match Sink::poll_ready(this.socket.as_mut(), cx) { Poll::Ready(_) => { @@ -281,6 +279,25 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static, } } +impl NotificationsOutSubstream { + /// Returns the number of items in the queue, capped to `u32::max_value()`. + pub fn queue_len(&self) -> u32 { + u32::try_from(self.messages_queue.len()).unwrap_or(u32::max_value()) + } + + /// Push a message to the queue of messages. + /// + /// This has the same effect as the `Sink::start_send` implementation. + pub fn push_message(&mut self, item: Vec) -> Result<(), NotificationsOutError> { + if self.messages_queue.len() >= MAX_PENDING_MESSAGES { + return Err(NotificationsOutError::Clogged); + } + + self.messages_queue.push_back(item); + Ok(()) + } +} + impl Sink> for NotificationsOutSubstream where TSubstream: AsyncRead + AsyncWrite + Unpin, { @@ -291,12 +308,7 @@ impl Sink> for NotificationsOutSubstream } fn start_send(mut self: Pin<&mut Self>, item: Vec) -> Result<(), Self::Error> { - if self.messages_queue.len() >= MAX_PENDING_MESSAGES { - return Err(NotificationsOutError::Clogged); - } - - self.messages_queue.push_back(item); - Ok(()) + self.push_message(item) } fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs index c96c5d08185735403978a22b7eecccaeb97a3dd8..85312b0803234e43dff213c4b1ade7f4f06a3c6c 100644 --- a/client/network/src/protocol/light_client_handler.rs +++ b/client/network/src/protocol/light_client_handler.rs @@ -29,7 +29,7 @@ use codec::{self, Encode, Decode}; use crate::{ chain::Client, config::ProtocolId, - protocol::{api, light_dispatch::TIMEOUT_REPUTATION_CHANGE} + protocol::{api, message::BlockAttributes} }; use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; use libp2p::{ @@ -37,6 +37,7 @@ use libp2p::{ ConnectedPoint, Multiaddr, PeerId, + connection::ConnectionId, upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, upgrade::{OutboundUpgrade, read_one, write_one} }, @@ -44,9 +45,11 @@ use libp2p::{ NegotiatedSubstream, NetworkBehaviour, NetworkBehaviourAction, + NotifyHandler, OneShotHandler, + OneShotHandlerConfig, PollParameters, - SubstreamProtocol + SubstreamProtocol, } }; use nohash_hasher::IntMap; @@ -58,6 +61,7 @@ use sp_core::{ storage::{ChildInfo, StorageKey}, hexdisplay::HexDisplay, }; +use smallvec::SmallVec; use sp_blockchain::{Error as ClientError}; use sp_runtime::{ traits::{Block, Header, NumberFor, Zero}, @@ -74,38 +78,52 @@ use std::{ use void::Void; use wasm_timer::Instant; +/// Reputation change for a peer when a request timed out. +pub(crate) const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); + /// Configuration options for `LightClientHandler` behaviour. #[derive(Debug, Clone)] pub struct Config { - max_data_size: usize, + max_request_size: usize, + max_response_size: usize, max_pending_requests: usize, inactivity_timeout: Duration, request_timeout: Duration, - protocol: Bytes, + light_protocol: Bytes, + block_protocol: Bytes, } impl Config { /// Create a fresh configuration with the following options: /// - /// - max. data size = 1 MiB + /// - max. request size = 1 MiB + /// - max. response size = 16 MiB /// - max. pending requests = 128 /// - inactivity timeout = 15s /// - request timeout = 15s pub fn new(id: &ProtocolId) -> Self { let mut c = Config { - max_data_size: 1024 * 1024, + max_request_size: 1 * 1024 * 1024, + max_response_size: 16 * 1024 * 1024, max_pending_requests: 128, inactivity_timeout: Duration::from_secs(15), request_timeout: Duration::from_secs(15), - protocol: Bytes::new(), + light_protocol: Bytes::new(), + block_protocol: Bytes::new(), }; c.set_protocol(id); c } - /// Limit the max. length of incoming request bytes. - pub fn set_max_data_size(&mut self, v: usize) -> &mut Self { - self.max_data_size = v; + /// Limit the max. length in bytes of a request. + pub fn set_max_request_size(&mut self, v: usize) -> &mut Self { + self.max_request_size = v; + self + } + + /// Limit the max. length in bytes of a response. + pub fn set_max_response_size(&mut self, v: usize) -> &mut Self { + self.max_response_size = v; self } @@ -129,11 +147,18 @@ impl Config { /// Set protocol to use for upgrade negotiation. pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { - let mut v = Vec::new(); - v.extend_from_slice(b"/"); - v.extend_from_slice(id.as_bytes()); - v.extend_from_slice(b"/light/2"); - self.protocol = v.into(); + let mut vl = Vec::new(); + vl.extend_from_slice(b"/"); + vl.extend_from_slice(id.as_bytes()); + vl.extend_from_slice(b"/light/2"); + self.light_protocol = vl.into(); + + let mut vb = Vec::new(); + vb.extend_from_slice(b"/"); + vb.extend_from_slice(id.as_bytes()); + vb.extend_from_slice(b"/sync/2"); + self.block_protocol = vb.into(); + self } } @@ -167,6 +192,10 @@ pub enum Error { // used because we currently only support a subset of those. #[derive(Debug)] pub enum Request { + Body { + request: fetcher::RemoteBodyRequest, + sender: oneshot::Sender, ClientError>> + }, Header { request: fetcher::RemoteHeaderRequest, sender: oneshot::Sender> @@ -199,7 +228,8 @@ enum Reply { VecU8(Vec), VecNumberU32(Vec<(::Number, u32)>), MapVecU8OptVecU8(HashMap, Option>>), - Header(B::Header) + Header(B::Header), + Extrinsics(Vec), } /// Augments a light client request with metadata. @@ -211,25 +241,39 @@ struct RequestWrapper { retries: usize, /// The actual request. request: Request, - /// Peer information, e.g. `PeerId`. - peer: P + /// The peer to send the request to, e.g. `PeerId`. + peer: P, + /// The connection to use for sending the request. + connection: Option, } /// Information we have about some peer. #[derive(Debug)] struct PeerInfo { - address: Multiaddr, + connections: SmallVec<[(ConnectionId, Multiaddr); crate::MAX_CONNECTIONS_PER_PEER]>, best_block: Option>, status: PeerStatus, } +impl Default for PeerInfo { + fn default() -> Self { + PeerInfo { + connections: SmallVec::new(), + best_block: None, + status: PeerStatus::Idle, + } + } +} + +type RequestId = u64; + /// A peer is either idle or busy processing a request from us. #[derive(Debug, Clone, PartialEq, Eq)] enum PeerStatus { /// The peer is available. Idle, /// We wait for the peer to return us a response for the given request ID. - BusyWith(u64), + BusyWith(RequestId), } /// The light client handler behaviour. @@ -247,9 +291,9 @@ pub struct LightClientHandler { /// Pending (local) requests. pending_requests: VecDeque>, /// Requests on their way to remote peers. - outstanding: IntMap>, + outstanding: IntMap>, /// (Local) Request ID counter - next_request_id: u64, + next_request_id: RequestId, /// Handle to use for reporting misbehaviour of peers. peerset: sc_peerset::PeersetHandle, } @@ -282,6 +326,7 @@ where /// means to determine it ourselves. pub fn update_best_block(&mut self, peer: &PeerId, num: NumberFor) { if let Some(info) = self.peers.get_mut(peer) { + log::trace!("new best block for {:?}: {:?}", peer, num); info.best_block = Some(num) } } @@ -296,35 +341,18 @@ where retries: retries(&req), request: req, peer: (), // we do not know the peer yet + connection: None, }; self.pending_requests.push_back(rw); Ok(()) } - fn next_request_id(&mut self) -> u64 { + fn next_request_id(&mut self) -> RequestId { let id = self.next_request_id; self.next_request_id += 1; id } - // Iterate over peers known to possess a certain block. - fn idle_peers_with_block(&mut self, num: NumberFor) -> impl Iterator + '_ { - self.peers.iter() - .filter(move |(_, info)| { - info.status == PeerStatus::Idle && info.best_block >= Some(num) - }) - .map(|(peer, _)| peer.clone()) - } - - // Iterate over peers without a known block. - fn idle_peers_with_unknown_block(&mut self) -> impl Iterator + '_ { - self.peers.iter() - .filter(|(_, info)| { - info.status == PeerStatus::Idle && info.best_block.is_none() - }) - .map(|(peer, _)| peer.clone()) - } - /// Remove the given peer. /// /// If we have a request to this peer in flight, we move it back to @@ -337,12 +365,50 @@ where retries: rw.retries, request: rw.request, peer: (), // need to find another peer + connection: None, }; self.pending_requests.push_back(rw); } self.peers.remove(peer); } + /// Prepares a request by selecting a suitable peer and connection to send it to. + /// + /// If there is currently no suitable peer for the request, the given request + /// is returned as `Err`. + fn prepare_request(&self, req: RequestWrapper) + -> Result<(PeerId, RequestWrapper), RequestWrapper> + { + let number = required_block(&req.request); + + let mut peer = None; + for (peer_id, peer_info) in self.peers.iter() { + if peer_info.status == PeerStatus::Idle { + match peer_info.best_block { + Some(n) => if n >= number { + peer = Some((peer_id, peer_info)); + break + }, + None => peer = Some((peer_id, peer_info)) + } + } + } + + if let Some((peer_id, peer_info)) = peer { + let connection = peer_info.connections.iter().next().map(|(id, _)| *id); + let rw = RequestWrapper { + timestamp: req.timestamp, + retries: req.retries, + request: req.request, + peer: peer_id.clone(), + connection, + }; + Ok((peer_id.clone(), rw)) + } else { + Err(req) + } + } + /// Process a local request's response from remote. /// /// If successful, this will give us the actual, checked data we should be @@ -351,10 +417,23 @@ where ( &mut self , peer: &PeerId , request: &Request - , response: api::v1::light::Response + , response: Response ) -> Result, Error> { log::trace!("response from {}", peer); + match response { + Response::Light(r) => self.on_response_light(peer, request, r), + Response::Block(r) => self.on_response_block(peer, request, r), + } + } + + fn on_response_light + ( &mut self + , peer: &PeerId + , request: &Request + , response: api::v1::light::Response + ) -> Result, Error> + { use api::v1::light::response::Response; match response.response { Some(Response::RemoteCallResponse(response)) => @@ -420,6 +499,32 @@ where } } + fn on_response_block + ( &mut self + , peer: &PeerId + , request: &Request + , response: api::v1::BlockResponse + ) -> Result, Error> + { + let request = if let Request::Body { request , .. } = &request { + request + } else { + return Err(Error::UnexpectedResponse); + }; + + let body: Vec<_> = match response.blocks.into_iter().next() { + Some(b) => b.body, + None => return Err(Error::UnexpectedResponse), + }; + + let body = body.into_iter() + .map(|mut extrinsic| B::Extrinsic::decode(&mut &extrinsic[..])) + .collect::>()?; + + let body = self.checker.check_body_proof(&request, body)?; + Ok(Reply::Extrinsics(body)) + } + fn on_remote_call_request ( &mut self , peer: &PeerId @@ -654,41 +759,71 @@ where fn new_handler(&mut self) -> Self::ProtocolsHandler { let p = InboundProtocol { - max_data_size: self.config.max_data_size, - protocol: self.config.protocol.clone(), + max_request_size: self.config.max_request_size, + protocol: self.config.light_protocol.clone(), }; - OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + let mut cfg = OneShotHandlerConfig::default(); + cfg.inactive_timeout = self.config.inactivity_timeout; + OneShotHandler::new(SubstreamProtocol::new(p), cfg) } fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec { self.peers.get(peer) - .map(|info| vec![info.address.clone()]) + .map(|info| info.connections.iter().map(|(_, a)| a.clone()).collect()) .unwrap_or_default() } - fn inject_connected(&mut self, peer: PeerId, info: ConnectedPoint) { + fn inject_connected(&mut self, peer: &PeerId) { + } + + fn inject_connection_established(&mut self, peer: &PeerId, conn: &ConnectionId, info: &ConnectedPoint) { let peer_address = match info { - ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, - ConnectedPoint::Dialer { address } => address + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr.clone(), + ConnectedPoint::Dialer { address } => address.clone() }; log::trace!("peer {} connected with address {}", peer, peer_address); - let info = PeerInfo { - address: peer_address, - best_block: None, - status: PeerStatus::Idle, - }; - - self.peers.insert(peer, info); + let entry = self.peers.entry(peer.clone()).or_default(); + entry.connections.push((*conn, peer_address)); } - fn inject_disconnected(&mut self, peer: &PeerId, _: ConnectedPoint) { + fn inject_disconnected(&mut self, peer: &PeerId) { log::trace!("peer {} disconnected", peer); self.remove_peer(peer) } - fn inject_node_event(&mut self, peer: PeerId, event: Event) { + fn inject_connection_closed(&mut self, peer: &PeerId, conn: &ConnectionId, info: &ConnectedPoint) { + let peer_address = match info { + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address } => address + }; + + log::trace!("connection to peer {} closed: {}", peer, peer_address); + + if let Some(info) = self.peers.get_mut(peer) { + info.connections.retain(|(c, _)| c != conn) + } + + // Add any outstanding requests on the closed connection back to the + // pending requests. + if let Some(id) = self.outstanding.iter() + .find(|(_, rw)| &rw.peer == peer && rw.connection == Some(*conn)) // (*) + .map(|(id, _)| *id) + { + let rw = self.outstanding.remove(&id).expect("by (*)"); + let rw = RequestWrapper { + timestamp: rw.timestamp, + retries: rw.retries, + request: rw.request, + peer: (), // need to find another peer + connection: None, + }; + self.pending_requests.push_back(rw); + } + } + + fn inject_event(&mut self, peer: PeerId, conn: ConnectionId, event: Event) { match event { // An incoming request from remote has been received. Event::Request(request, mut stream) => { @@ -734,9 +869,10 @@ where // A response to one of our own requests has been received. Event::Response(id, response) => { if let Some(request) = self.outstanding.remove(&id) { - // We first just check if the response originates from the expected peer. + // We first just check if the response originates from the expected peer + // and connection. if request.peer != peer { - log::debug!("was expecting response from {} instead of {}", request.peer, peer); + log::debug!("Expected response from {} instead of {}.", request.peer, peer); self.outstanding.insert(id, request); self.remove_peer(&peer); self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); @@ -768,6 +904,7 @@ where retries: request.retries, request: request.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw); } @@ -781,6 +918,7 @@ where retries: request.retries - 1, request: request.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw) } else { @@ -820,44 +958,54 @@ where request.timestamp = Instant::now(); request.retries -= 1 } - let number = required_block(&request.request); - let available_peer = { - let p = self.idle_peers_with_block(number).next(); - if p.is_none() { - self.idle_peers_with_unknown_block().next() - } else { - p + + + match self.prepare_request(request) { + Err(request) => { + self.pending_requests.push_front(request); + log::debug!("no peer available to send request to"); + break } - }; - if let Some(peer) = available_peer { - let rq = serialize_request(&request.request); - let mut buf = Vec::with_capacity(rq.encoded_len()); - if let Err(e) = rq.encode(&mut buf) { - log::debug!("failed to serialize request: {}", e); - send_reply(Err(ClientError::RemoteFetchFailed), request.request) - } else { - let id = self.next_request_id(); - log::trace!("sending request {} to peer {}", id, peer); - let protocol = OutboundProtocol { - request: buf, - request_id: id, - max_data_size: self.config.max_data_size, - protocol: self.config.protocol.clone(), + Ok((peer, request)) => { + let request_bytes = match serialize_request(&request.request) { + Ok(bytes) => bytes, + Err(error) => { + log::debug!("failed to serialize request: {}", error); + send_reply(Err(ClientError::RemoteFetchFailed), request.request); + continue + } }; - self.peers.get_mut(&peer).map(|info| info.status = PeerStatus::BusyWith(id)); - let rw = RequestWrapper { - timestamp: request.timestamp, - retries: request.retries, - request: request.request, - peer: peer.clone(), + + let (expected, protocol) = match request.request { + Request::Body { .. } => + (ExpectedResponseTy::Block, self.config.block_protocol.clone()), + _ => + (ExpectedResponseTy::Light, self.config.light_protocol.clone()), }; - self.outstanding.insert(id, rw); - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id: peer, event: protocol }) + + let peer_id = peer.clone(); + let handler = request.connection.map_or(NotifyHandler::Any, NotifyHandler::One); + + let request_id = self.next_request_id(); + self.peers.get_mut(&peer).map(|p| p.status = PeerStatus::BusyWith(request_id)); + self.outstanding.insert(request_id, request); + + let event = OutboundProtocol { + request_id, + request: request_bytes, + expected, + max_response_size: self.config.max_response_size, + protocol, + }; + + log::trace!("sending request {} to peer {}", request_id, peer_id); + + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { + peer_id, + handler, + event, + }) } - } else { - self.pending_requests.push_front(request); - log::debug!("no peer available to send request to"); - break } } @@ -883,6 +1031,7 @@ where retries: rw.retries - 1, request: rw.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw) } @@ -894,6 +1043,7 @@ where fn required_block(request: &Request) -> NumberFor { match request { + Request::Body { request, .. } => *request.header.number(), Request::Header { request, .. } => request.block, Request::Read { request, .. } => *request.header.number(), Request::ReadChild { request, .. } => *request.header.number(), @@ -904,6 +1054,7 @@ fn required_block(request: &Request) -> NumberFor { fn retries(request: &Request) -> usize { let rc = match request { + Request::Body { request, .. } => request.retry_count, Request::Header { request, .. } => request.retry_count, Request::Read { request, .. } => request.retry_count, Request::ReadChild { request, .. } => request.retry_count, @@ -913,8 +1064,20 @@ fn retries(request: &Request) -> usize { rc.unwrap_or(0) } -fn serialize_request(request: &Request) -> api::v1::light::Request { +fn serialize_request(request: &Request) -> Result, prost::EncodeError> { let request = match request { + Request::Body { request, .. } => { + let rq = api::v1::BlockRequest { + fields: u32::from(BlockAttributes::BODY.bits()), + from_block: Some(api::v1::block_request::FromBlock::Hash(request.header.hash().encode())), + to_block: Vec::new(), + direction: api::v1::Direction::Ascending as i32, + max_blocks: 1, + }; + let mut buf = Vec::with_capacity(rq.encoded_len()); + rq.encode(&mut buf)?; + return Ok(buf); + } Request::Header { request, .. } => { let r = api::v1::light::RemoteHeaderRequest { block: request.block.encode() }; api::v1::light::request::Request::RemoteHeaderRequest(r) @@ -957,7 +1120,10 @@ fn serialize_request(request: &Request) -> api::v1::light::Request } }; - api::v1::light::Request { request: Some(request) } + let rq = api::v1::light::Request { request: Some(request) }; + let mut buf = Vec::with_capacity(rq.encoded_len()); + rq.encode(&mut buf)?; + Ok(buf) } fn send_reply(result: Result, ClientError>, request: Request) { @@ -965,6 +1131,11 @@ fn send_reply(result: Result, ClientError>, request: Request< let _ = sender.send(item); // It is okay if the other end already hung up. } match request { + Request::Body { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::Extrinsics(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for body request: {:?}, {:?}", reply, request), + } Request::Header { request, sender } => match result { Err(e) => send(Err(e), sender), Ok(Reply::Header(x)) => send(Ok(x), sender), @@ -999,7 +1170,16 @@ pub enum Event { /// Incoming request from remote and substream to use for the response. Request(api::v1::light::Request, T), /// Incoming response from remote. - Response(u64, api::v1::light::Response), + Response(RequestId, Response), +} + +/// Incoming response from remote. +#[derive(Debug, Clone)] +pub enum Response { + /// Incoming light response from remote. + Light(api::v1::light::Response), + /// Incoming block response from remote. + Block(api::v1::BlockResponse), } /// Substream upgrade protocol. @@ -1008,31 +1188,31 @@ pub enum Event { #[derive(Debug, Clone)] pub struct InboundProtocol { /// The max. request length in bytes. - max_data_size: usize, + max_request_size: usize, /// The protocol to use for upgrade negotiation. protocol: Bytes, } impl UpgradeInfo for InboundProtocol { - type Info = Bytes; - type InfoIter = iter::Once; + type Info = Bytes; + type InfoIter = iter::Once; - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } } impl InboundUpgrade for InboundProtocol where T: AsyncRead + AsyncWrite + Unpin + Send + 'static { - type Output = Event; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; - fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { let future = async move { - let vec = read_one(&mut s, self.max_data_size).await?; + let vec = read_one(&mut s, self.max_request_size).await?; match api::v1::light::Request::decode(&vec[..]) { Ok(r) => Ok(Event::Request(r, s)), Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) @@ -1050,39 +1230,60 @@ pub struct OutboundProtocol { /// The serialized protobuf request. request: Vec, /// Local identifier for the request. Used to associate it with a response. - request_id: u64, - /// The max. request length in bytes. - max_data_size: usize, + request_id: RequestId, + /// Kind of response expected for this request. + expected: ExpectedResponseTy, + /// The max. response length in bytes. + max_response_size: usize, /// The protocol to use for upgrade negotiation. protocol: Bytes, } +/// Type of response expected from the remote for this request. +#[derive(Debug, Clone)] +enum ExpectedResponseTy { + Light, + Block, +} + impl UpgradeInfo for OutboundProtocol { - type Info = Bytes; - type InfoIter = iter::Once; + type Info = Bytes; + type InfoIter = iter::Once; - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } } impl OutboundUpgrade for OutboundProtocol where T: AsyncRead + AsyncWrite + Unpin + Send + 'static { - type Output = Event; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; - fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { + fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { let future = async move { write_one(&mut s, &self.request).await?; - let vec = read_one(&mut s, self.max_data_size).await?; - api::v1::light::Response::decode(&vec[..]) - .map(|r| Event::Response(self.request_id, r)) - .map_err(|e| { - ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) - }) + let vec = read_one(&mut s, self.max_response_size).await?; + + match self.expected { + ExpectedResponseTy::Light => { + api::v1::light::Response::decode(&vec[..]) + .map(|r| Event::Response(self.request_id, Response::Light(r))) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + }, + ExpectedResponseTy::Block => { + api::v1::BlockResponse::decode(&vec[..]) + .map(|r| Event::Response(self.request_id, Response::Block(r))) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + } + } }; future.boxed() } @@ -1108,7 +1309,7 @@ mod tests { use crate::{ chain::Client, config::ProtocolId, - protocol::{api, light_dispatch::tests::{DummyFetchChecker, dummy_header}} + protocol::api, }; use futures::{channel::oneshot, prelude::*}; use libp2p::{ @@ -1116,6 +1317,7 @@ mod tests { Multiaddr, core::{ ConnectedPoint, + connection::ConnectionId, identity, muxing::{StreamMuxerBox, SubstreamRef}, transport::{Transport, boxed::Boxed, memory::MemoryTransport}, @@ -1130,15 +1332,15 @@ mod tests { use sp_blockchain::{Error as ClientError}; use sp_core::storage::ChildInfo; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, io, iter::{self, FromIterator}, pin::Pin, sync::Arc, task::{Context, Poll} }; - use sp_runtime::{generic::Header, traits::BlakeTwo256}; - use super::{Event, LightClientHandler, Request, OutboundProtocol, PeerStatus}; + use sp_runtime::{generic::Header, traits::{BlakeTwo256, Block as BlockT, NumberFor}}; + use super::{Event, LightClientHandler, Request, Response, OutboundProtocol, PeerStatus}; use void::Void; const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"foobarbaz"); @@ -1153,7 +1355,7 @@ mod tests { fn make_swarm(ok: bool, ps: sc_peerset::PeersetHandle, cf: super::Config) -> Swarm { let client = Arc::new(substrate_test_runtime_client::new()); - let checker = Arc::new(DummyFetchChecker::new(ok)); + let checker = Arc::new(DummyFetchChecker { ok, _mark: std::marker::PhantomData }); let id_key = identity::Keypair::generate_ed25519(); let dh_key = Keypair::::new().into_authentic(&id_key).unwrap(); let local_peer = id_key.public().into_peer_id(); @@ -1167,10 +1369,104 @@ mod tests { Swarm::new(transport, LightClientHandler::new(cf, client, checker, ps), local_peer) } + struct DummyFetchChecker { + ok: bool, + _mark: std::marker::PhantomData + } + + impl fetcher::FetchChecker for DummyFetchChecker { + fn check_header_proof( + &self, + _request: &fetcher::RemoteHeaderRequest, + header: Option, + _remote_proof: fetcher::StorageProof, + ) -> Result { + match self.ok { + true if header.is_some() => Ok(header.unwrap()), + _ => Err(ClientError::Backend("Test error".into())), + } + } + + fn check_read_proof( + &self, + request: &fetcher::RemoteReadRequest, + _: fetcher::StorageProof, + ) -> Result, Option>>, ClientError> { + match self.ok { + true => Ok(request.keys + .iter() + .cloned() + .map(|k| (k, Some(vec![42]))) + .collect() + ), + false => Err(ClientError::Backend("Test error".into())), + } + } + + fn check_read_child_proof( + &self, + request: &fetcher::RemoteReadChildRequest, + _: fetcher::StorageProof, + ) -> Result, Option>>, ClientError> { + match self.ok { + true => Ok(request.keys + .iter() + .cloned() + .map(|k| (k, Some(vec![42]))) + .collect() + ), + false => Err(ClientError::Backend("Test error".into())), + } + } + + fn check_execution_proof( + &self, + _: &fetcher::RemoteCallRequest, + _: fetcher::StorageProof, + ) -> Result, ClientError> { + match self.ok { + true => Ok(vec![42]), + false => Err(ClientError::Backend("Test error".into())), + } + } + + fn check_changes_proof( + &self, + _: &fetcher::RemoteChangesRequest, + _: fetcher::ChangesProof + ) -> Result, u32)>, ClientError> { + match self.ok { + true => Ok(vec![(100.into(), 2)]), + false => Err(ClientError::Backend("Test error".into())), + } + } + + fn check_body_proof( + &self, + _: &fetcher::RemoteBodyRequest, + body: Vec + ) -> Result, ClientError> { + match self.ok { + true => Ok(body), + false => Err(ClientError::Backend("Test error".into())), + } + } + } + fn make_config() -> super::Config { super::Config::new(&ProtocolId::from(&b"foo"[..])) } + fn dummy_header() -> sp_test_primitives::Header { + sp_test_primitives::Header { + parent_hash: Default::default(), + number: 0, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + } + } + struct EmptyPollParams(PeerId); impl PollParameters for EmptyPollParams { @@ -1201,7 +1497,7 @@ mod tests { out_peers: 128, bootnodes: Vec::new(), reserved_only: false, - reserved_nodes: Vec::new(), + priority_groups: Vec::new(), }; sc_peerset::Peerset::from_config(cfg) } @@ -1213,7 +1509,7 @@ mod tests { ) -> LightClientHandler { let client = Arc::new(substrate_test_runtime_client::new()); - let checker = Arc::new(DummyFetchChecker::new(ok)); + let checker = Arc::new(DummyFetchChecker { ok, _mark: std::marker::PhantomData }); LightClientHandler::new(cf, client, checker, ps) } @@ -1235,10 +1531,12 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + behaviour.inject_connection_established(&peer, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); - behaviour.inject_disconnected(&peer, empty_dialer()); + behaviour.inject_connection_closed(&peer, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_disconnected(&peer); assert_eq!(0, behaviour.peers.len()) } @@ -1249,8 +1547,10 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer0.clone(), empty_dialer()); - behaviour.inject_connected(peer1.clone(), empty_dialer()); + behaviour.inject_connection_established(&peer0, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_connected(&peer0); + behaviour.inject_connection_established(&peer1, &ConnectionId::new(2), &empty_dialer()); + behaviour.inject_connected(&peer1); // We now know about two peers. assert_eq!(HashSet::from_iter(&[peer0.clone(), peer1.clone()]), behaviour.peers.keys().collect::>()); @@ -1272,7 +1572,7 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); // The behaviour should now attempt to send the request. - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, .. }) => { + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, .. }) => { assert!(peer_id == peer0 || peer_id == peer1) }); @@ -1312,7 +1612,9 @@ mod tests { let mut behaviour = make_behaviour(false, pset.1, make_config()); // ^--- Making sure the response data check fails. - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let chan = oneshot::channel(); @@ -1340,7 +1642,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(request_id, response)); + behaviour.inject_event(peer.clone(), conn, Event::Response(request_id, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1356,7 +1658,9 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); @@ -1369,7 +1673,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(2347895932, response)); + behaviour.inject_event(peer.clone(), conn, Event::Response(2347895932, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); @@ -1383,7 +1687,9 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let chan = oneshot::channel(); @@ -1411,7 +1717,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(request_id, response)); + behaviour.inject_event(peer.clone(), conn, Event::Response(request_id, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1431,10 +1737,18 @@ mod tests { let mut behaviour = make_behaviour(false, pset.1, make_config()); // ^--- Making sure the response data check fails. - behaviour.inject_connected(peer1.clone(), empty_dialer()); - behaviour.inject_connected(peer2.clone(), empty_dialer()); - behaviour.inject_connected(peer3.clone(), empty_dialer()); - behaviour.inject_connected(peer4.clone(), empty_dialer()); + let conn1 = ConnectionId::new(1); + behaviour.inject_connection_established(&peer1, &conn1, &empty_dialer()); + behaviour.inject_connected(&peer1); + let conn2 = ConnectionId::new(2); + behaviour.inject_connection_established(&peer2, &conn2, &empty_dialer()); + behaviour.inject_connected(&peer2); + let conn3 = ConnectionId::new(3); + behaviour.inject_connection_established(&peer3, &conn3, &empty_dialer()); + behaviour.inject_connected(&peer3); + let conn4 = ConnectionId::new(3); + behaviour.inject_connection_established(&peer4, &conn4, &empty_dialer()); + behaviour.inject_connected(&peer4); assert_eq!(4, behaviour.peers.len()); let mut chan = oneshot::channel(); @@ -1449,11 +1763,11 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(1, behaviour.outstanding.len()); - for _ in 0 .. 3 { + for i in 1 ..= 3 { // Construct an invalid response let request_id = *behaviour.outstanding.keys().next().unwrap(); let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); @@ -1463,8 +1777,9 @@ mod tests { response: Some(api::v1::light::response::Response::RemoteCallResponse(r)) } }; - behaviour.inject_node_event(responding_peer, Event::Response(request_id, response.clone())); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + let conn = ConnectionId::new(i); + behaviour.inject_event(responding_peer, conn, Event::Response(request_id, Response::Light(response.clone()))); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_matches!(chan.1.try_recv(), Ok(None)) } // Final invalid response @@ -1476,7 +1791,7 @@ mod tests { response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } }; - behaviour.inject_node_event(responding_peer, Event::Response(request_id, response)); + behaviour.inject_event(responding_peer, conn4, Event::Response(request_id, Response::Light(response))); assert_matches!(poll(&mut behaviour), Poll::Pending); assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) } @@ -1486,10 +1801,13 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let response = match request { + Request::Body { .. } => unimplemented!(), Request::Header{..} => { let r = api::v1::light::RemoteHeaderResponse { header: dummy_header().encode(), @@ -1534,12 +1852,12 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(1, behaviour.outstanding.len()); assert_eq!(1, *behaviour.outstanding.keys().next().unwrap()); - behaviour.inject_node_event(peer.clone(), Event::Response(1, response)); + behaviour.inject_event(peer.clone(), conn, Event::Response(1, Response::Light(response))); poll(&mut behaviour); diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs deleted file mode 100644 index 22d26075b3ceb8654dcbac03e7cc3c9fc2bf4f9a..0000000000000000000000000000000000000000 --- a/client/network/src/protocol/light_dispatch.rs +++ /dev/null @@ -1,1313 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Light client requests service. -//! -//! Handles requests for data coming from our local light client and that must be answered by -//! nodes on the network. - -use std::collections::{HashMap, VecDeque}; -use std::sync::Arc; -use std::time::Duration; -use wasm_timer::Instant; -use log::{trace, info}; -use futures::channel::oneshot::{Sender as OneShotSender}; -use linked_hash_map::{Entry, LinkedHashMap}; -use sp_blockchain::Error as ClientError; -use sc_client_api::{FetchChecker, RemoteHeaderRequest, - RemoteCallRequest, RemoteReadRequest, RemoteChangesRequest, ChangesProof, - RemoteReadChildRequest, RemoteBodyRequest, StorageProof}; -use crate::protocol::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; -use libp2p::PeerId; -use crate::config::Roles; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use sc_peerset::ReputationChange; - -/// Remote request timeout. -const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); -/// Default request retry count. -const RETRY_COUNT: usize = 1; -/// Reputation change for a peer when a request timed out. -pub(crate) const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); - -/// Trait used by the `LightDispatch` service to communicate messages back to the network. -pub trait LightDispatchNetwork { - /// Adjusts the reputation of the given peer. - fn report_peer(&mut self, who: &PeerId, reputation_change: ReputationChange); - - /// Disconnect from the given peer. Used in case of misbehaviour. - fn disconnect_peer(&mut self, who: &PeerId); - - /// Send to `who` a request for a header. - fn send_header_request(&mut self, who: &PeerId, id: RequestId, block: <::Header as HeaderT>::Number); - - /// Send to `who` a read request. - fn send_read_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - keys: Vec>, - ); - - /// Send to `who` a child read request. - fn send_read_child_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - storage_key: Vec, - child_info: Vec, - child_type: u32, - keys: Vec>, - ); - - /// Send to `who` a call request. - fn send_call_request( - &mut self, - who: &PeerId, - id: RequestId, - block: ::Hash, - method: String, - data: Vec - ); - - /// Send to `who` a changes request. - fn send_changes_request( - &mut self, - who: &PeerId, - id: RequestId, - first: ::Hash, - last: ::Hash, - min: ::Hash, - max: ::Hash, - storage_key: Option>, - key: Vec, - ); - - /// Send to `who` a body request. - fn send_body_request( - &mut self, - who: &PeerId, - id: RequestId, - fields: BlockAttributes, - from: FromBlock<::Hash, <::Header as HeaderT>::Number>, - to: Option, - direction: Direction, - max: Option - ); -} - -/// Light client requests service. Dispatches requests to appropriate peers. -pub struct LightDispatch { - /// Verifies that responses are correct. Passed at initialization. - checker: Arc>, - /// Numeric ID to assign to the next outgoing request. Used to assign responses to their - /// corresponding request. - next_request_id: u64, - /// Requests that we have yet to send out on the network. - pending_requests: VecDeque>, - /// List of nodes to which we have sent a request and that are yet to answer. - active_peers: LinkedHashMap>, - /// List of nodes that we know of that aren't doing anything and that are available for new - /// requests. - idle_peers: VecDeque, - /// Best known block for each node in `active_peers` and `idle_peers`. - best_blocks: HashMap>, -} - -struct Request { - id: u64, - /// When the request got created or sent out to the network. - timestamp: Instant, - /// Number of remaining attempts to fulfill this request. If it reaches 0, we interrupt the - /// attempt. - retry_count: usize, - data: RequestData, -} - -/// One request for data made by the `Client`. -/// -/// Contains a `Sender` where to send the result. -pub(crate) enum RequestData { - RemoteBody(RemoteBodyRequest, OneShotSender, ClientError>>), - RemoteHeader(RemoteHeaderRequest, OneShotSender>), - RemoteRead( - RemoteReadRequest, - OneShotSender, Option>>, ClientError>>, - ), - RemoteReadChild( - RemoteReadChildRequest, - OneShotSender, Option>>, ClientError>> - ), - RemoteCall(RemoteCallRequest, OneShotSender, ClientError>>), - RemoteChanges( - RemoteChangesRequest, - OneShotSender, u32)>, ClientError>> - ), -} - -enum Accept { - Ok, - CheckFailed(ClientError, RequestData), - Unexpected(RequestData), -} - -/// Dummy implementation of `FetchChecker` that always assumes that responses are bad. -/// -/// Considering that it is the responsibility of the client to build the fetcher, it can use this -/// implementation if it knows that it will never perform any request. -#[derive(Default, Clone)] -pub struct AlwaysBadChecker; - -impl FetchChecker for AlwaysBadChecker { - fn check_header_proof( - &self, - _request: &RemoteHeaderRequest, - _remote_header: Option, - _remote_proof: StorageProof, - ) -> Result { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } - - fn check_read_proof( - &self, - _request: &RemoteReadRequest, - _remote_proof: StorageProof, - ) -> Result,Option>>, ClientError> { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } - - fn check_read_child_proof( - &self, - _request: &RemoteReadChildRequest, - _remote_proof: StorageProof, - ) -> Result, Option>>, ClientError> { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } - - fn check_execution_proof( - &self, - _request: &RemoteCallRequest, - _remote_proof: StorageProof, - ) -> Result, ClientError> { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } - - fn check_changes_proof( - &self, - _request: &RemoteChangesRequest, - _remote_proof: ChangesProof - ) -> Result, u32)>, ClientError> { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } - - fn check_body_proof( - &self, - _request: &RemoteBodyRequest, - _body: Vec - ) -> Result, ClientError> { - Err(ClientError::Msg("AlwaysBadChecker".into())) - } -} - -impl LightDispatch where - B::Header: HeaderT, -{ - /// Creates new light client requests processor. - pub fn new(checker: Arc>) -> Self { - LightDispatch { - checker, - next_request_id: 0, - pending_requests: VecDeque::new(), - active_peers: LinkedHashMap::new(), - idle_peers: VecDeque::new(), - best_blocks: HashMap::new(), - } - } - - /// Inserts a new request in the list of requests to execute. - pub(crate) fn add_request(&mut self, network: impl LightDispatchNetwork, data: RequestData) { - self.insert(RETRY_COUNT, data); - self.dispatch(network); - } - - /// Inserts a new request in the list of requests to execute. - fn insert(&mut self, retry_count: usize, data: RequestData) { - let request_id = self.next_request_id; - self.next_request_id += 1; - - self.pending_requests.push_back(Request { - id: request_id, - timestamp: Instant::now(), - retry_count, - data, - }); - } - - /// Try to accept response from given peer. - fn accept_response( - &mut self, - rtype: &str, - mut network: impl LightDispatchNetwork, - peer: PeerId, - request_id: u64, - try_accept: impl FnOnce(Request, &Arc>) -> Accept - ) { - let request = match self.remove(peer.clone(), request_id) { - Some(request) => request, - None => { - info!("Invalid remote {} response from peer {}", rtype, peer); - network.report_peer(&peer, ReputationChange::new_fatal("Invalid remote response")); - network.disconnect_peer(&peer); - self.remove_peer(&peer); - return; - }, - }; - - let retry_count = request.retry_count; - let (retry_count, retry_request_data) = match try_accept(request, &self.checker) { - Accept::Ok => (retry_count, None), - Accept::CheckFailed(error, retry_request_data) => { - info!("Failed to check remote {} response from peer {}: {}", rtype, peer, error); - network.report_peer(&peer, ReputationChange::new_fatal("Failed remote response check")); - network.disconnect_peer(&peer); - self.remove_peer(&peer); - - if retry_count > 0 { - (retry_count - 1, Some(retry_request_data)) - } else { - trace!(target: "sync", "Failed to get remote {} response for given number of retries", rtype); - retry_request_data.fail(ClientError::RemoteFetchFailed.into()); - (0, None) - } - }, - Accept::Unexpected(retry_request_data) => { - info!("Unexpected response to remote {} from peer", rtype); - network.report_peer(&peer, ReputationChange::new_fatal("Unexpected remote response")); - network.disconnect_peer(&peer); - self.remove_peer(&peer); - - (retry_count, Some(retry_request_data)) - }, - }; - - if let Some(request_data) = retry_request_data { - self.insert(retry_count, request_data); - } - - self.dispatch(network); - } - - /// Call this when we connect to a node on the network. - pub fn on_connect( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - role: Roles, - best_number: NumberFor - ) { - if !role.is_full() { - return; - } - - self.idle_peers.push_back(peer.clone()); - self.best_blocks.insert(peer, best_number); - - self.dispatch(network); - } - - /// Sets the best seen block for the given node. - pub fn update_best_number(&mut self, network: impl LightDispatchNetwork, peer: PeerId, best_number: NumberFor) { - self.best_blocks.insert(peer, best_number); - self.dispatch(network); - } - - /// Call this when we disconnect from a node. - pub fn on_disconnect(&mut self, network: impl LightDispatchNetwork, peer: &PeerId) { - self.remove_peer(peer); - self.dispatch(network); - } - - /// Must be called periodically in order to perform maintenance. - pub fn maintain_peers(&mut self, mut network: impl LightDispatchNetwork) { - let now = Instant::now(); - - loop { - match self.active_peers.front() { - Some((_, request)) if now - request.timestamp >= REQUEST_TIMEOUT => (), - _ => break, - } - - let (bad_peer, request) = self.active_peers.pop_front().expect("front() is Some as checked above"); - self.pending_requests.push_front(request); - network.report_peer(&bad_peer, ReputationChange::new(TIMEOUT_REPUTATION_CHANGE, "Light request timeout")); - network.disconnect_peer(&bad_peer); - } - - self.dispatch(network); - } - - /// Handles a remote header response message from on the network. - pub fn on_remote_header_response( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - response: message::RemoteHeaderResponse - ) { - self.accept_response("header", network, peer, response.id, |request, checker| match request.data { - RequestData::RemoteHeader(request, sender) => match checker.check_header_proof( - &request, - response.header, - response.proof - ) { - Ok(response) => { - // we do not bother if receiver has been dropped already - let _ = sender.send(Ok(response)); - Accept::Ok - }, - Err(error) => Accept::CheckFailed(error, RequestData::RemoteHeader(request, sender)), - }, - data => Accept::Unexpected(data), - }) - } - - /// Handles a remote read response message from on the network. - pub fn on_remote_read_response( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - response: message::RemoteReadResponse - ) { - self.accept_response("read", network, peer, response.id, |request, checker| match request.data { - RequestData::RemoteRead(request, sender) => { - match checker.check_read_proof(&request, response.proof) { - Ok(response) => { - // we do not bother if receiver has been dropped already - let _ = sender.send(Ok(response)); - Accept::Ok - }, - Err(error) => Accept::CheckFailed( - error, - RequestData::RemoteRead(request, sender) - ), - }}, - RequestData::RemoteReadChild(request, sender) => { - match checker.check_read_child_proof(&request, response.proof) { - Ok(response) => { - // we do not bother if receiver has been dropped already - let _ = sender.send(Ok(response)); - Accept::Ok - }, - Err(error) => Accept::CheckFailed( - error, - RequestData::RemoteReadChild(request, sender) - ), - }}, - data => Accept::Unexpected(data), - }) - } - - /// Handles a remote call response message from on the network. - pub fn on_remote_call_response( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - response: message::RemoteCallResponse - ) { - self.accept_response("call", network, peer, response.id, |request, checker| match request.data { - RequestData::RemoteCall(request, sender) => match checker.check_execution_proof(&request, response.proof) { - Ok(response) => { - // we do not bother if receiver has been dropped already - let _ = sender.send(Ok(response)); - Accept::Ok - }, - Err(error) => Accept::CheckFailed(error, RequestData::RemoteCall(request, sender)), - }, - data => Accept::Unexpected(data), - }) - } - - /// Handles a remote changes response message from on the network. - pub fn on_remote_changes_response( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - response: message::RemoteChangesResponse, B::Hash> - ) { - self.accept_response("changes", network, peer, response.id, |request, checker| match request.data { - RequestData::RemoteChanges(request, sender) => match checker.check_changes_proof( - &request, ChangesProof { - max_block: response.max, - proof: response.proof, - roots: response.roots.into_iter().collect(), - roots_proof: response.roots_proof, - }) { - Ok(response) => { - // we do not bother if receiver has been dropped already - let _ = sender.send(Ok(response)); - Accept::Ok - }, - Err(error) => Accept::CheckFailed(error, RequestData::RemoteChanges(request, sender)), - }, - data => Accept::Unexpected(data), - }) - } - - /// Handles a remote body response message from on the network. - pub fn on_remote_body_response( - &mut self, - network: impl LightDispatchNetwork, - peer: PeerId, - response: message::BlockResponse - ) { - self.accept_response("body", network, peer, response.id, |request, checker| match request.data { - RequestData::RemoteBody(request, sender) => { - let mut bodies: Vec<_> = response - .blocks - .into_iter() - .filter_map(|b| b.body) - .collect(); - - // Number of bodies are hardcoded to 1 for valid `RemoteBodyResponses` - if bodies.len() != 1 { - return Accept::CheckFailed( - "RemoteBodyResponse: invalid number of blocks".into(), - RequestData::RemoteBody(request, sender), - ) - } - let body = bodies.remove(0); - - match checker.check_body_proof(&request, body) { - Ok(body) => { - let _ = sender.send(Ok(body)); - Accept::Ok - } - Err(error) => Accept::CheckFailed(error, RequestData::RemoteBody(request, sender)), - } - } - other => Accept::Unexpected(other), - }) - } - - pub fn is_light_response(&self, peer: &PeerId, request_id: message::RequestId) -> bool { - self.active_peers.get(peer).map_or(false, |r| r.id == request_id) - } - - fn remove(&mut self, peer: PeerId, id: u64) -> Option> { - match self.active_peers.entry(peer.clone()) { - Entry::Occupied(entry) => match entry.get().id == id { - true => { - self.idle_peers.push_back(peer); - Some(entry.remove()) - }, - false => None, - }, - Entry::Vacant(_) => None, - } - } - - /// Removes a peer from the list of known peers. - /// - /// Puts back the active request that this node was performing into `pending_requests`. - fn remove_peer(&mut self, peer: &PeerId) { - self.best_blocks.remove(peer); - - if let Some(request) = self.active_peers.remove(peer) { - self.pending_requests.push_front(request); - return; - } - - if let Some(idle_index) = self.idle_peers.iter().position(|i| i == peer) { - self.idle_peers.swap_remove_back(idle_index); - } - } - - /// Dispatches pending requests. - fn dispatch(&mut self, mut network: impl LightDispatchNetwork) { - let mut last_peer = self.idle_peers.back().cloned(); - let mut unhandled_requests = VecDeque::new(); - - loop { - let peer = match self.idle_peers.pop_front() { - Some(peer) => peer, - None => break, - }; - - // check if request can (optimistically) be processed by the peer - let can_be_processed_by_peer = { - let request = match self.pending_requests.front() { - Some(r) => r, - None => { - self.idle_peers.push_front(peer); - break; - }, - }; - let peer_best_block = self.best_blocks.get(&peer) - .expect("entries are inserted into best_blocks when peer is connected; - entries are removed from best_blocks when peer is disconnected; - peer is in idle_peers and thus connected; qed"); - request.required_block() <= *peer_best_block - }; - - if !can_be_processed_by_peer { - // return peer to the back of the queue - self.idle_peers.push_back(peer.clone()); - - // we have enumerated all peers and no one can handle request - if Some(peer) == last_peer { - let request = self.pending_requests.pop_front().expect("checked in loop condition; qed"); - unhandled_requests.push_back(request); - last_peer = self.idle_peers.back().cloned(); - } - - continue; - } - - last_peer = self.idle_peers.back().cloned(); - - let mut request = self.pending_requests.pop_front().expect("checked in loop condition; qed"); - request.timestamp = Instant::now(); - trace!(target: "sync", "Dispatching remote request {} to peer {}", request.id, peer); - request.send_to(&mut network, &peer); - self.active_peers.insert(peer, request); - } - - self.pending_requests.append(&mut unhandled_requests); - } -} - -impl Request { - /// Returns the block that the remote needs to have in order to be able to fulfill - /// this request. - fn required_block(&self) -> NumberFor { - match self.data { - RequestData::RemoteHeader(ref data, _) => data.block, - RequestData::RemoteRead(ref data, _) => *data.header.number(), - RequestData::RemoteReadChild(ref data, _) => *data.header.number(), - RequestData::RemoteCall(ref data, _) => *data.header.number(), - RequestData::RemoteChanges(ref data, _) => data.max_block.0, - RequestData::RemoteBody(ref data, _) => *data.header.number(), - } - } - - fn send_to(&self, out: &mut impl LightDispatchNetwork, peer: &PeerId) { - match self.data { - RequestData::RemoteHeader(ref data, _) => - out.send_header_request( - peer, - self.id, - data.block, - ), - RequestData::RemoteRead(ref data, _) => - out.send_read_request( - peer, - self.id, - data.block, - data.keys.clone(), - ), - RequestData::RemoteReadChild(ref data, _) => - out.send_read_child_request( - peer, - self.id, - data.block, - data.storage_key.clone(), - data.child_info.clone(), - data.child_type, - data.keys.clone(), - ), - RequestData::RemoteCall(ref data, _) => - out.send_call_request( - peer, - self.id, - data.block, - data.method.clone(), - data.call_data.clone(), - ), - RequestData::RemoteChanges(ref data, _) => - out.send_changes_request( - peer, - self.id, - data.first_block.1.clone(), - data.last_block.1.clone(), - data.tries_roots.1.clone(), - data.max_block.1.clone(), - data.storage_key.clone(), - data.key.clone(), - ), - RequestData::RemoteBody(ref data, _) => - out.send_body_request( - peer, - self.id, - message::BlockAttributes::BODY, - message::FromBlock::Hash(data.header.hash()), - None, - message::Direction::Ascending, - Some(1) - ), - } - } -} - -impl RequestData { - fn fail(self, error: ClientError) { - // don't care if anyone is listening - match self { - RequestData::RemoteHeader(_, sender) => { let _ = sender.send(Err(error)); }, - RequestData::RemoteCall(_, sender) => { let _ = sender.send(Err(error)); }, - RequestData::RemoteRead(_, sender) => { let _ = sender.send(Err(error)); }, - RequestData::RemoteReadChild(_, sender) => { let _ = sender.send(Err(error)); }, - RequestData::RemoteChanges(_, sender) => { let _ = sender.send(Err(error)); }, - RequestData::RemoteBody(_, sender) => { let _ = sender.send(Err(error)); }, - } - } -} - -#[cfg(test)] -pub mod tests { - use std::collections::{HashMap, HashSet}; - use std::sync::Arc; - use std::time::Instant; - use futures::channel::oneshot; - use sp_core::storage::ChildInfo; - use sp_runtime::traits::{Block as BlockT, NumberFor, Header as HeaderT}; - use sp_blockchain::{Error as ClientError, Result as ClientResult}; - use sc_client_api::{FetchChecker, RemoteHeaderRequest, - ChangesProof, RemoteCallRequest, RemoteReadRequest, - RemoteReadChildRequest, RemoteChangesRequest, RemoteBodyRequest}; - use crate::config::Roles; - use crate::protocol::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; - use libp2p::PeerId; - use super::{REQUEST_TIMEOUT, LightDispatch, LightDispatchNetwork, RequestData, StorageProof}; - use sp_test_primitives::{Block, Header}; - - pub(crate) struct DummyFetchChecker { - pub(crate) ok: bool, - _mark: std::marker::PhantomData - } - - impl DummyFetchChecker { - pub(crate) fn new(ok: bool) -> Self { - DummyFetchChecker { ok, _mark: std::marker::PhantomData } - } - } - - impl FetchChecker for DummyFetchChecker { - fn check_header_proof( - &self, - _request: &RemoteHeaderRequest, - header: Option, - _remote_proof: StorageProof, - ) -> ClientResult { - match self.ok { - true if header.is_some() => Ok(header.unwrap()), - _ => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_read_proof( - &self, - request: &RemoteReadRequest, - _: StorageProof, - ) -> ClientResult, Option>>> { - match self.ok { - true => Ok(request.keys - .iter() - .cloned() - .map(|k| (k, Some(vec![42]))) - .collect() - ), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_read_child_proof( - &self, - request: &RemoteReadChildRequest, - _: StorageProof, - ) -> ClientResult, Option>>> { - match self.ok { - true => Ok(request.keys - .iter() - .cloned() - .map(|k| (k, Some(vec![42]))) - .collect() - ), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_execution_proof( - &self, - _: &RemoteCallRequest, - _: StorageProof, - ) -> ClientResult> { - match self.ok { - true => Ok(vec![42]), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_changes_proof( - &self, - _: &RemoteChangesRequest, - _: ChangesProof - ) -> ClientResult, u32)>> { - match self.ok { - true => Ok(vec![(100.into(), 2)]), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_body_proof( - &self, - _: &RemoteBodyRequest, - body: Vec - ) -> ClientResult> { - match self.ok { - true => Ok(body), - false => Err(ClientError::Backend("Test error".into())), - } - } - } - - fn dummy(ok: bool) -> LightDispatch { - LightDispatch::new(Arc::new(DummyFetchChecker::new(ok))) - } - - fn total_peers(light_dispatch: &LightDispatch) -> usize { - light_dispatch.idle_peers.len() + light_dispatch.active_peers.len() - } - - fn receive_call_response( - network_interface: impl LightDispatchNetwork, - light_dispatch: &mut LightDispatch, - peer: PeerId, - id: message::RequestId - ) { - light_dispatch.on_remote_call_response(network_interface, peer, message::RemoteCallResponse { - id: id, - proof: StorageProof::empty(), - }); - } - - pub(crate) fn dummy_header() -> Header { - Header { - parent_hash: Default::default(), - number: 0, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - - #[derive(Default)] - struct DummyNetwork { - disconnected_peers: HashSet, - } - - impl<'a, B: BlockT> LightDispatchNetwork for &'a mut DummyNetwork { - fn report_peer(&mut self, _: &PeerId, _: crate::ReputationChange) {} - fn disconnect_peer(&mut self, who: &PeerId) { - self.disconnected_peers.insert(who.clone()); - } - fn send_header_request(&mut self, _: &PeerId, _: RequestId, _: <::Header as HeaderT>::Number) {} - fn send_read_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec>) {} - fn send_read_child_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: Vec, - _: Vec, _: u32, _: Vec>) {} - fn send_call_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: String, _: Vec) {} - fn send_changes_request(&mut self, _: &PeerId, _: RequestId, _: ::Hash, _: ::Hash, - _: ::Hash, _: ::Hash, _: Option>, _: Vec) {} - fn send_body_request(&mut self, _: &PeerId, _: RequestId, _: BlockAttributes, _: FromBlock<::Hash, - <::Header as HeaderT>::Number>, _: Option, _: Direction, _: Option) {} - } - - fn assert_disconnected_peer(dummy: &DummyNetwork) { - assert_eq!(dummy.disconnected_peers.len(), 1); - } - - #[test] - fn knows_about_peers_roles() { - let mut network_interface = DummyNetwork::default(); - let mut light_dispatch = dummy(true); - let peer0 = PeerId::random(); - let peer1 = PeerId::random(); - let peer2 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0, Roles::LIGHT, 1000); - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 2000); - light_dispatch.on_connect(&mut network_interface, peer2.clone(), Roles::AUTHORITY, 3000); - assert_eq!(vec![peer1.clone(), peer2.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert_eq!(light_dispatch.best_blocks.get(&peer1), Some(&2000)); - assert_eq!(light_dispatch.best_blocks.get(&peer2), Some(&3000)); - } - - #[test] - fn disconnects_from_idle_peer() { - let peer0 = PeerId::random(); - - let mut network_interface = DummyNetwork::default(); - let mut light_dispatch = dummy(true); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 100); - assert_eq!(1, total_peers(&light_dispatch)); - assert!(!light_dispatch.best_blocks.is_empty()); - - light_dispatch.on_disconnect(&mut network_interface, &peer0); - assert_eq!(0, total_peers(&light_dispatch)); - assert!(light_dispatch.best_blocks.is_empty()); - } - - #[test] - fn disconnects_from_timeouted_peer() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - let peer1 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 1000); - assert_eq!(vec![peer0.clone(), peer1.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert!(light_dispatch.active_peers.is_empty()); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: None, - }, oneshot::channel().0)); - assert_eq!(vec![peer1.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert_eq!(vec![peer0.clone()], light_dispatch.active_peers.keys().cloned().collect::>()); - - light_dispatch.active_peers[&peer0].timestamp = Instant::now() - REQUEST_TIMEOUT - REQUEST_TIMEOUT; - light_dispatch.maintain_peers(&mut network_interface); - assert!(light_dispatch.idle_peers.is_empty()); - assert_eq!(vec![peer1.clone()], light_dispatch.active_peers.keys().cloned().collect::>()); - assert_disconnected_peer(&network_interface); - } - - #[test] - fn disconnects_from_peer_on_response_with_wrong_id() { - let mut light_dispatch = dummy(true); - let peer0 = PeerId::random(); - let mut network_interface = DummyNetwork::default(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: None, - }, oneshot::channel().0)); - receive_call_response(&mut network_interface, &mut light_dispatch, peer0, 1); - assert_disconnected_peer(&network_interface); - assert_eq!(light_dispatch.pending_requests.len(), 1); - } - - #[test] - fn disconnects_from_peer_on_incorrect_response() { - let mut light_dispatch = dummy(false); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }, oneshot::channel().0)); - - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - receive_call_response(&mut network_interface, &mut light_dispatch, peer0.clone(), 0); - assert_disconnected_peer(&network_interface); - assert_eq!(light_dispatch.pending_requests.len(), 1); - } - - #[test] - fn disconnects_from_peer_on_unexpected_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - receive_call_response(&mut network_interface, &mut light_dispatch, peer0, 0); - assert_disconnected_peer(&network_interface); - } - - #[test] - fn disconnects_from_peer_on_wrong_response_type() { - let mut light_dispatch = dummy(false); - let peer0 = PeerId::random(); - let mut network_interface = DummyNetwork::default(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }, oneshot::channel().0)); - - light_dispatch.on_remote_read_response(&mut network_interface, peer0.clone(), message::RemoteReadResponse { - id: 0, - proof: StorageProof::empty(), - }); - assert_disconnected_peer(&network_interface); - assert_eq!(light_dispatch.pending_requests.len(), 1); - } - - #[test] - fn receives_remote_failure_after_retry_count_failures() { - let retry_count = 2; - let peer_ids = (0 .. retry_count + 1).map(|_| PeerId::random()).collect::>(); - let mut light_dispatch = dummy(false); - let mut network_interface = DummyNetwork::default(); - for i in 0..retry_count+1 { - light_dispatch.on_connect(&mut network_interface, peer_ids[i].clone(), Roles::FULL, 1000); - } - - let (tx, mut response) = oneshot::channel(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(retry_count) - }, tx)); - - for i in 0..retry_count { - assert!(response.try_recv().unwrap().is_none()); - receive_call_response(&mut network_interface, &mut light_dispatch, peer_ids[i].clone(), i as u64); - } - - assert!(response.try_recv().unwrap().unwrap().is_err()); - } - - #[test] - fn receives_remote_call_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - let (tx, response) = oneshot::channel(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteCall(RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: None, - }, tx)); - - receive_call_response(&mut network_interface, &mut light_dispatch, peer0.clone(), 0); - assert_eq!(futures::executor::block_on(response).unwrap().unwrap(), vec![42]); - } - - #[test] - fn receives_remote_read_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - let (tx, response) = oneshot::channel(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteRead(RemoteReadRequest { - header: dummy_header(), - block: Default::default(), - keys: vec![b":key".to_vec()], - retry_count: None, - }, tx)); - - light_dispatch.on_remote_read_response(&mut network_interface, peer0.clone(), message::RemoteReadResponse { - id: 0, - proof: StorageProof::empty(), - }); - assert_eq!( - futures::executor::block_on(response).unwrap().unwrap().remove(b":key".as_ref()).unwrap(), - Some(vec![42]) - ); - } - - #[test] - fn receives_remote_read_child_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - let (tx, response) = oneshot::channel(); - let child_info = ChildInfo::new_default(b"unique_id_1"); - let (child_info, child_type) = child_info.info(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteReadChild(RemoteReadChildRequest { - header: dummy_header(), - block: Default::default(), - storage_key: b":child_storage:sub".to_vec(), - child_info: child_info.to_vec(), - child_type, - keys: vec![b":key".to_vec()], - retry_count: None, - }, tx)); - - light_dispatch.on_remote_read_response(&mut network_interface, - peer0.clone(), message::RemoteReadResponse { - id: 0, - proof: StorageProof::empty(), - }); - assert_eq!(futures::executor::block_on(response).unwrap().unwrap().remove(b":key".as_ref()).unwrap(), Some(vec![42])); - } - - #[test] - fn receives_remote_header_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - let (tx, response) = oneshot::channel(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 1, - retry_count: None, - }, tx)); - - light_dispatch.on_remote_header_response(&mut network_interface, peer0.clone(), message::RemoteHeaderResponse { - id: 0, - header: Some(Header { - parent_hash: Default::default(), - number: 1, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - }), - proof: StorageProof::empty(), - }); - assert_eq!( - futures::executor::block_on(response).unwrap().unwrap().hash(), - "6443a0b46e0412e626363028115a9f2cf963eeed526b8b33e5316f08b50d0dc3".parse().unwrap(), - ); - } - - #[test] - fn receives_remote_changes_response() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer0 = PeerId::random(); - light_dispatch.on_connect(&mut network_interface, peer0.clone(), Roles::FULL, 1000); - - let (tx, response) = oneshot::channel(); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteChanges(RemoteChangesRequest { - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), - }], - first_block: (1, Default::default()), - last_block: (100, Default::default()), - max_block: (100, Default::default()), - tries_roots: (1, Default::default(), vec![]), - key: vec![], - storage_key: None, - retry_count: None, - }, tx)); - - light_dispatch.on_remote_changes_response(&mut network_interface, peer0.clone(), message::RemoteChangesResponse { - id: 0, - max: 1000, - proof: vec![vec![2]], - roots: vec![], - roots_proof: StorageProof::empty(), - }); - assert_eq!(futures::executor::block_on(response).unwrap().unwrap(), vec![(100, 2)]); - } - - #[test] - fn does_not_sends_request_to_peer_who_has_no_required_block() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer1 = PeerId::random(); - let peer2 = PeerId::random(); - - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 100); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 200, - retry_count: None, - }, oneshot::channel().0)); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 250, - retry_count: None, - }, oneshot::channel().0)); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 250, - retry_count: None, - }, oneshot::channel().0)); - - light_dispatch.on_connect(&mut network_interface, peer2.clone(), Roles::FULL, 150); - - assert_eq!(vec![peer1.clone(), peer2.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert_eq!(light_dispatch.pending_requests.len(), 3); - - light_dispatch.update_best_number(&mut network_interface, peer1.clone(), 250); - - assert_eq!(vec![peer2.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert_eq!(light_dispatch.pending_requests.len(), 2); - - light_dispatch.update_best_number(&mut network_interface, peer2.clone(), 250); - - assert!(!light_dispatch.idle_peers.iter().any(|_| true)); - assert_eq!(light_dispatch.pending_requests.len(), 1); - - light_dispatch.on_remote_header_response(&mut network_interface, peer1.clone(), message::RemoteHeaderResponse { - id: 0, - header: Some(dummy_header()), - proof: StorageProof::empty(), - }); - - assert!(!light_dispatch.idle_peers.iter().any(|_| true)); - assert_eq!(light_dispatch.pending_requests.len(), 0); - } - - #[test] - fn does_not_loop_forever_after_dispatching_request_to_last_peer() { - // this test is a regression for a bug where the dispatch function would - // loop forever after dispatching a request to the last peer, since the - // last peer was not updated - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer1 = PeerId::random(); - let peer2 = PeerId::random(); - let peer3 = PeerId::random(); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 250, - retry_count: None, - }, oneshot::channel().0)); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 250, - retry_count: None, - }, oneshot::channel().0)); - - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 200); - light_dispatch.on_connect(&mut network_interface, peer2.clone(), Roles::FULL, 200); - light_dispatch.on_connect(&mut network_interface, peer3.clone(), Roles::FULL, 250); - - assert_eq!(vec![peer1.clone(), peer2.clone()], light_dispatch.idle_peers.iter().cloned().collect::>()); - assert_eq!(light_dispatch.pending_requests.len(), 1); - } - - #[test] - fn tries_to_send_all_pending_requests() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer1 = PeerId::random(); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 300, - retry_count: None, - }, oneshot::channel().0)); - light_dispatch.add_request(&mut network_interface, RequestData::RemoteHeader(RemoteHeaderRequest { - cht_root: Default::default(), - block: 250, - retry_count: None, - }, oneshot::channel().0)); - - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 250); - - assert!(light_dispatch.idle_peers.iter().cloned().collect::>().is_empty()); - assert_eq!(light_dispatch.pending_requests.len(), 1); - } - - #[test] - fn remote_body_with_one_block_body_should_succeed() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer1 = PeerId::random(); - - let header = dummy_header(); - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 250); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteBody(RemoteBodyRequest { - header: header.clone(), - retry_count: None, - }, oneshot::channel().0)); - - assert!(light_dispatch.pending_requests.is_empty()); - assert_eq!(light_dispatch.active_peers.len(), 1); - - let block = message::BlockData:: { - hash: sp_core::H256::random(), - header: None, - body: Some(Vec::new()), - message_queue: None, - receipt: None, - justification: None, - }; - - let response = message::generic::BlockResponse { - id: 0, - blocks: vec![block], - }; - - light_dispatch.on_remote_body_response(&mut network_interface, peer1.clone(), response); - - assert!(light_dispatch.active_peers.is_empty()); - assert_eq!(light_dispatch.idle_peers.len(), 1); - } - - #[test] - fn remote_body_with_three_bodies_should_fail() { - let mut light_dispatch = dummy(true); - let mut network_interface = DummyNetwork::default(); - let peer1 = PeerId::random(); - - let header = dummy_header(); - light_dispatch.on_connect(&mut network_interface, peer1.clone(), Roles::FULL, 250); - - light_dispatch.add_request(&mut network_interface, RequestData::RemoteBody(RemoteBodyRequest { - header: header.clone(), - retry_count: None, - }, oneshot::channel().0)); - - assert!(light_dispatch.pending_requests.is_empty()); - assert_eq!(light_dispatch.active_peers.len(), 1); - - let response = { - let blocks: Vec<_> = (0..3).map(|_| message::BlockData:: { - hash: sp_core::H256::random(), - header: None, - body: Some(Vec::new()), - message_queue: None, - receipt: None, - justification: None, - }).collect(); - - message::generic::BlockResponse { - id: 0, - blocks, - } - }; - - light_dispatch.on_remote_body_response(&mut network_interface, peer1.clone(), response); - assert!(light_dispatch.active_peers.is_empty()); - assert!(light_dispatch.idle_peers.is_empty(), "peer should be disconnected after bad response"); - } -} diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index a12c26da2e47eec14ae136ede05e6094d9e0c715..ae83b49e60f57e475bb3399ec6af3e9e672e6c8f 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -24,7 +24,7 @@ pub use self::generic::{ RemoteHeaderRequest, RemoteHeaderResponse, RemoteChangesRequest, RemoteChangesResponse, FinalityProofRequest, FinalityProofResponse, - FromBlock, RemoteReadChildRequest, + FromBlock, RemoteReadChildRequest, Roles, }; use sc_client_api::StorageProof; @@ -137,14 +137,71 @@ pub struct RemoteReadResponse { /// Generic types. pub mod generic { + use bitflags::bitflags; use codec::{Encode, Decode, Input, Output}; use sp_runtime::Justification; - use crate::config::Roles; use super::{ RemoteReadResponse, Transactions, Direction, RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId, BlockState, StorageProof, }; + + bitflags! { + /// Bitmask of the roles that a node fulfills. + pub struct Roles: u8 { + /// No network. + const NONE = 0b00000000; + /// Full node, does not participate in consensus. + const FULL = 0b00000001; + /// Light client node. + const LIGHT = 0b00000010; + /// Act as an authority + const AUTHORITY = 0b00000100; + } + } + + impl Roles { + /// Does this role represents a client that holds full chain data locally? + pub fn is_full(&self) -> bool { + self.intersects(Roles::FULL | Roles::AUTHORITY) + } + + /// Does this role represents a client that does not participates in the consensus? + pub fn is_authority(&self) -> bool { + *self == Roles::AUTHORITY + } + + /// Does this role represents a client that does not hold full chain data locally? + pub fn is_light(&self) -> bool { + !self.is_full() + } + } + + impl<'a> From<&'a crate::config::Role> for Roles { + fn from(roles: &'a crate::config::Role) -> Self { + match roles { + crate::config::Role::Full => Roles::FULL, + crate::config::Role::Light => Roles::LIGHT, + crate::config::Role::Sentry { .. } => Roles::AUTHORITY, + crate::config::Role::Authority { .. } => Roles::AUTHORITY, + } + } + } + + impl codec::Encode for Roles { + fn encode_to(&self, dest: &mut T) { + dest.push_byte(self.bits()) + } + } + + impl codec::EncodeLike for Roles {} + + impl codec::Decode for Roles { + fn decode(input: &mut I) -> Result { + Self::from_bits(input.read_byte()?).ok_or_else(|| codec::Error::from("Invalid bytes")) + } + } + /// Consensus is mostly opaque to us #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct ConsensusMessage { diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 683f3c3139cc7ea86bb828cb63472cda2c3b0509..9feded784fe78871646879ba49e3b19a866a6d7c 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -34,9 +34,9 @@ use sp_consensus::{BlockOrigin, BlockStatus, import_queue::{IncomingBlock, BlockImportResult, BlockImportError} }; use crate::{ - config::{Roles, BoxFinalityProofRequestBuilder}, + config::BoxFinalityProofRequestBuilder, protocol::message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, - FinalityProofResponse}, + FinalityProofResponse, Roles}, }; use either::Either; use extra_requests::ExtraRequests; @@ -378,12 +378,12 @@ impl ChainSync { Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) } Ok(BlockStatus::KnownBad) => { - info!("New peer with known bad best block {} ({}).", best_hash, best_number); + info!("💔 New peer with known bad best block {} ({}).", best_hash, best_number); Err(BadPeer(who, rep::BAD_BLOCK)) } Ok(BlockStatus::Unknown) => { if best_number.is_zero() { - info!("New peer with unknown genesis hash {} ({}).", best_hash, best_number); + info!("💔 New peer with unknown genesis hash {} ({}).", best_hash, best_number); return Err(BadPeer(who, rep::GENESIS_MISMATCH)); } // If there are more than `MAJOR_SYNC_BLOCKS` in the import queue then we have @@ -711,7 +711,7 @@ impl ChainSync { return Err(BadPeer(who, rep::UNKNOWN_ANCESTOR)) }, (_, Err(e)) => { - info!("Error answering legitimate blockchain query: {:?}", e); + info!("❌ Error answering legitimate blockchain query: {:?}", e); return Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) } }; @@ -818,7 +818,7 @@ impl ChainSync { if let Some(peer) = self.peers.get_mut(&who) { peer } else { - error!(target: "sync", "Called on_block_justification with a bad peer ID"); + error!(target: "sync", "💔 Called on_block_justification with a bad peer ID"); return Ok(OnBlockJustification::Nothing) }; @@ -831,7 +831,7 @@ impl ChainSync { if hash != block.hash { info!( target: "sync", - "Invalid block justification provided by {}: requested: {:?} got: {:?}", who, hash, block.hash + "💔 Invalid block justification provided by {}: requested: {:?} got: {:?}", who, hash, block.hash ); return Err(BadPeer(who, rep::BAD_JUSTIFICATION)); } @@ -865,7 +865,7 @@ impl ChainSync { if let Some(peer) = self.peers.get_mut(&who) { peer } else { - error!(target: "sync", "Called on_block_finality_proof_data with a bad peer ID"); + error!(target: "sync", "💔 Called on_block_finality_proof_data with a bad peer ID"); return Ok(OnBlockFinalityProof::Nothing) }; @@ -877,7 +877,7 @@ impl ChainSync { if hash != resp.block { info!( target: "sync", - "Invalid block finality proof provided: requested: {:?} got: {:?}", + "💔 Invalid block finality proof provided: requested: {:?} got: {:?}", hash, resp.block ); @@ -943,7 +943,7 @@ impl ChainSync { if aux.bad_justification { if let Some(peer) = who { - info!("Sent block with bad justification to import"); + info!("💔 Sent block with bad justification to import"); output.push(Err(BadPeer(peer, rep::BAD_JUSTIFICATION))); } } @@ -959,21 +959,21 @@ impl ChainSync { }, Err(BlockImportError::IncompleteHeader(who)) => { if let Some(peer) = who { - warn!("Peer sent block with incomplete header to import"); + warn!("💔 Peer sent block with incomplete header to import"); output.push(Err(BadPeer(peer, rep::INCOMPLETE_HEADER))); output.extend(self.restart()); } }, Err(BlockImportError::VerificationFailed(who, e)) => { if let Some(peer) = who { - warn!("Verification failed for block {:?} received from peer: {}, {:?}", hash, peer, e); + warn!("💔 Verification failed for block {:?} received from peer: {}, {:?}", hash, peer, e); output.push(Err(BadPeer(peer, rep::VERIFICATION_FAIL))); output.extend(self.restart()); } }, Err(BlockImportError::BadBlock(who)) => { if let Some(peer) = who { - info!("Block {:?} received from peer {} has been blacklisted", hash, peer); + info!("💔 Block {:?} received from peer {} has been blacklisted", hash, peer); output.push(Err(BadPeer(peer, rep::BAD_BLOCK))); } }, @@ -985,7 +985,7 @@ impl ChainSync { }, e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { - warn!(target: "sync", "Error importing block {:?}: {:?}", hash, e); + warn!(target: "sync", "💔 Error importing block {:?}: {:?}", hash, e); output.extend(self.restart()); }, Err(BlockImportError::Cancelled) => {} @@ -1017,7 +1017,7 @@ impl ChainSync { }); if let Err(err) = r { - warn!(target: "sync", "Error cleaning up pending extra finality proof data requests: {:?}", err) + warn!(target: "sync", "💔 Error cleaning up pending extra finality proof data requests: {:?}", err) } let client = &self.client; @@ -1026,7 +1026,7 @@ impl ChainSync { }); if let Err(err) = r { - warn!(target: "sync", "Error cleaning up pending extra justification data requests: {:?}", err); + warn!(target: "sync", "💔 Error cleaning up pending extra justification data requests: {:?}", err); } } @@ -1080,7 +1080,7 @@ impl ChainSync { let number = *header.number(); debug!(target: "sync", "Received block announcement {:?} with number {:?} from {}", hash, number, who); if number.is_zero() { - warn!(target: "sync", "Ignored genesis block (#0) announcement from {}: {}", who, hash); + warn!(target: "sync", "💔 Ignored genesis block (#0) announcement from {}: {}", who, hash); return OnBlockAnnounce::Nothing } let parent_status = self.block_status(header.parent_hash()).ok().unwrap_or(BlockStatus::Unknown); @@ -1091,7 +1091,7 @@ impl ChainSync { let peer = if let Some(peer) = self.peers.get_mut(&who) { peer } else { - error!(target: "sync", "Called on_block_announce with a bad peer ID"); + error!(target: "sync", "💔 Called on_block_announce with a bad peer ID"); return OnBlockAnnounce::Nothing }; while peer.recently_announced.len() >= ANNOUNCE_HISTORY_SIZE { @@ -1108,10 +1108,12 @@ impl ChainSync { } // If the announced block is the best they have and is not ahead of us, our common number // is either one further ahead or it's the one they just announced, if we know about it. - if is_best && self.best_queued_number >= number { - if known { + if is_best { + if known && self.best_queued_number >= number { peer.common_number = number - } else if header.parent_hash() == &self.best_queued_hash || known_parent { + } else if header.parent_hash() == &self.best_queued_hash + || known_parent && self.best_queued_number >= number + { peer.common_number = number - One::one(); } } @@ -1135,7 +1137,7 @@ impl ChainSync { return OnBlockAnnounce::Nothing } Err(e) => { - error!(target: "sync", "Block announcement validation errored: {}", e); + error!(target: "sync", "💔 Block announcement validation errored: {}", e); return OnBlockAnnounce::Nothing } } @@ -1320,13 +1322,17 @@ fn peer_block_request( finalized: NumberFor, best_num: NumberFor, ) -> Option<(Range>, BlockRequest)> { - if peer.common_number < finalized { - return None; - } if best_num >= peer.best_number { // Will be downloaded as alternative fork instead. return None; } + if peer.common_number < finalized { + trace!( + target: "sync", + "Requesting pre-finalized chain from {:?}, common={}, finalized={}, peer best={}, our best={}", + id, finalized, peer.common_number, peer.best_number, best_num, + ); + } if let Some(range) = blocks.needed_blocks( id.clone(), MAX_BLOCKS_TO_REQUEST, diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 81b12a1a704aab9b17737c4a03190bef519ff3c5..3d854b574b01fc26aaf6d1d39e15bed2b46551c2 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -221,7 +221,7 @@ impl ExtraRequests { }; if self.tree.finalize_root(&finalized_hash).is_none() { - warn!(target: "sync", "Imported {:?} {:?} which isn't a root in the tree: {:?}", + warn!(target: "sync", "‼️ Imported {:?} {:?} which isn't a root in the tree: {:?}", finalized_hash, finalized_number, self.tree.roots().collect::>() diff --git a/client/network/src/service.rs b/client/network/src/service.rs index e06d3bd13e8a693e5ea5c54890fdbad62dcd7c29..48b5c30d3ef44e2f341d2f00775d89ede849f9c1 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -25,31 +25,44 @@ //! The methods of the [`NetworkService`] are implemented by sending a message over a channel, //! which is then processed by [`NetworkWorker::poll`]. -use std::{borrow::Cow, collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path, str}; -use std::sync::{Arc, atomic::{AtomicBool, AtomicUsize, Ordering}}; -use std::pin::Pin; -use std::task::Poll; - -use sp_consensus::import_queue::{ImportQueue, Link}; -use sp_consensus::import_queue::{BlockImportResult, BlockImportError}; -use futures::{prelude::*, channel::mpsc}; -use log::{warn, error, info, trace}; -use libp2p::{PeerId, Multiaddr, kad::record}; -use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent}; +use crate::{ + behaviour::{Behaviour, BehaviourOut}, + config::{parse_addr, parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig}, + error::Error, + network_state::{ + NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer, + }, + on_demand_layer::AlwaysBadChecker, + protocol::{self, event::Event, light_client_handler, sync::SyncState, PeerInfo, Protocol}, + transport, ReputationChange, +}; +use futures::prelude::*; +use libp2p::{PeerId, Multiaddr}; +use libp2p::core::{ConnectedPoint, Executor, connection::{ConnectionError, PendingConnectionError}}; +use libp2p::kad::record; +use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent, protocols_handler::NodeHandlerWrapperError}; +use log::{error, info, trace, warn}; use parking_lot::Mutex; +use prometheus_endpoint::{ + register, Counter, CounterVec, Gauge, GaugeVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, U64, +}; use sc_peerset::PeersetHandle; -use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId}; -use prometheus_endpoint::{Registry, Counter, CounterVec, Gauge, GaugeVec, Opts, U64, register, PrometheusError}; - -use crate::{behaviour::{Behaviour, BehaviourOut}, config::{parse_str_addr, parse_addr}}; -use crate::{transport, config::NonReservedPeerMode, ReputationChange}; -use crate::config::{Params, TransportConfig}; -use crate::error::Error; -use crate::network_state::{NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer}; -use crate::protocol::{self, Protocol, PeerInfo}; -use crate::protocol::{event::Event, light_dispatch::{AlwaysBadChecker, RequestData}}; -use crate::protocol::sync::SyncState; - +use sp_consensus::import_queue::{BlockImportError, BlockImportResult, ImportQueue, Link}; +use sp_runtime::{ + traits::{Block as BlockT, NumberFor}, + ConsensusEngineId, +}; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + fs, io, + marker::PhantomData, + pin::Pin, + str, + sync::{atomic::{AtomicBool, AtomicUsize, Ordering}, Arc}, + task::Poll, +}; /// Minimum Requirements for a Hash within Networking pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} @@ -148,7 +161,7 @@ pub struct NetworkService { /// nodes it should be connected to or not. peerset: PeersetHandle, /// Channel that sends messages to the actual worker. - to_worker: mpsc::UnboundedSender>, + to_worker: TracingUnboundedSender>, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, @@ -161,28 +174,20 @@ impl NetworkWorker { /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. pub fn new(params: Params) -> Result, Error> { - let (to_worker, from_worker) = mpsc::unbounded(); + let (to_worker, from_worker) = tracing_unbounded("mpsc_network_worker"); - if let Some(ref path) = params.network_config.net_config_path { - fs::create_dir_all(Path::new(path))?; - } + fs::create_dir_all(¶ms.network_config.net_config_path)?; // List of multiaddresses that we know in the network. let mut known_addresses = Vec::new(); let mut bootnodes = Vec::new(); - let mut reserved_nodes = Vec::new(); let mut boot_node_ids = HashSet::new(); // Process the bootnodes. for bootnode in params.network_config.boot_nodes.iter() { - match parse_str_addr(bootnode) { - Ok((peer_id, addr)) => { - bootnodes.push(peer_id.clone()); - boot_node_ids.insert(peer_id.clone()); - known_addresses.push((peer_id, addr)); - }, - Err(_) => warn!(target: "sub-libp2p", "Not a valid bootnode address: {}", bootnode), - } + bootnodes.push(bootnode.peer_id.clone()); + boot_node_ids.insert(bootnode.peer_id.clone()); + known_addresses.push((bootnode.peer_id.clone(), bootnode.multiaddr.clone())); } let boot_node_ids = Arc::new(boot_node_ids); @@ -204,29 +209,56 @@ impl NetworkWorker { } )?; - // Initialize the reserved peers. - for reserved in params.network_config.reserved_nodes.iter() { - if let Ok((peer_id, addr)) = parse_str_addr(reserved) { - reserved_nodes.push(peer_id.clone()); - known_addresses.push((peer_id, addr)); - } else { - warn!(target: "sub-libp2p", "Not a valid reserved node address: {}", reserved); + // Initialize the peers we should always be connected to. + let priority_groups = { + let mut reserved_nodes = HashSet::new(); + for reserved in params.network_config.reserved_nodes.iter() { + reserved_nodes.insert(reserved.peer_id.clone()); + known_addresses.push((reserved.peer_id.clone(), reserved.multiaddr.clone())); } - } + + let mut sentries_and_validators = HashSet::new(); + match ¶ms.role { + Role::Sentry { validators } => { + for validator in validators { + sentries_and_validators.insert(validator.peer_id.clone()); + known_addresses.push((validator.peer_id.clone(), validator.multiaddr.clone())); + } + } + Role::Authority { sentry_nodes } => { + for sentry_node in sentry_nodes { + sentries_and_validators.insert(sentry_node.peer_id.clone()); + known_addresses.push((sentry_node.peer_id.clone(), sentry_node.multiaddr.clone())); + } + } + _ => {} + } + + vec![ + ("reserved".to_owned(), reserved_nodes), + ("sentries_and_validators".to_owned(), sentries_and_validators), + ] + }; let peerset_config = sc_peerset::PeersetConfig { in_peers: params.network_config.in_peers, out_peers: params.network_config.out_peers, bootnodes, reserved_only: params.network_config.non_reserved_mode == NonReservedPeerMode::Deny, - reserved_nodes, + priority_groups, }; // Private and public keys configuration. let local_identity = params.network_config.node_key.clone().into_keypair()?; let local_public = local_identity.public(); let local_peer_id = local_public.clone().into_peer_id(); - info!(target: "sub-libp2p", "Local node identity is: {}", local_peer_id.to_base58()); + info!(target: "sub-libp2p", "🏷 Local node identity is: {}", local_peer_id.to_base58()); + + // Initialize the metrics. + let metrics = match ¶ms.metrics_registry { + Some(registry) => Some(Metrics::register(®istry)?), + None => None + }; let checker = params.on_demand.as_ref() .map(|od| od.checker().clone()) @@ -236,11 +268,10 @@ impl NetworkWorker { let is_major_syncing = Arc::new(AtomicBool::new(false)); let (protocol, peerset_handle) = Protocol::new( protocol::ProtocolConfig { - roles: params.roles, + roles: From::from(¶ms.role), max_parallel_downloads: params.network_config.max_parallel_downloads, }, params.chain.clone(), - checker.clone(), params.transaction_pool, params.finality_proof_provider.clone(), params.finality_proof_request_builder, @@ -249,6 +280,7 @@ impl NetworkWorker { params.block_announce_validator, params.metrics_registry.as_ref(), boot_node_ids.clone(), + metrics.as_ref().map(|m| m.notifications_queues_size.clone()), )?; // Build the swarm. @@ -268,6 +300,7 @@ impl NetworkWorker { }; let behaviour = futures::executor::block_on(Behaviour::new( protocol, + params.role, user_agent, local_public, known_addresses, @@ -291,9 +324,16 @@ impl NetworkWorker { }; transport::build_transport(local_identity, config_mem, config_wasm, flowctrl) }; - let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()); + let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()) + .peer_connection_limit(crate::MAX_CONNECTIONS_PER_PEER); if let Some(spawner) = params.executor { - builder = builder.executor_fn(spawner); + struct SpawnImpl(F); + impl + Send>>)> Executor for SpawnImpl { + fn exec(&self, f: Pin + Send>>) { + (self.0)(f) + } + } + builder = builder.executor(Box::new(SpawnImpl(spawner))); } (builder.build(), bandwidth) }; @@ -333,10 +373,7 @@ impl NetworkWorker { from_worker, light_client_rqs: params.on_demand.and_then(|od| od.extract_receiver()), event_streams: Vec::new(), - metrics: match params.metrics_registry { - Some(registry) => Some(Metrics::register(®istry)?), - None => None - }, + metrics, boot_node_ids, }) } @@ -403,8 +440,8 @@ impl NetworkWorker { } /// You must call this when a new block is imported by the client. - pub fn on_block_imported(&mut self, header: B::Header, data: Vec, is_best: bool) { - self.network_service.user_protocol_mut().on_block_imported(&header, data, is_best); + pub fn on_block_imported(&mut self, header: B::Header, is_best: bool) { + self.network_service.user_protocol_mut().on_block_imported(&header, is_best); } /// You must call this when a new block is finalized by the client. @@ -520,7 +557,7 @@ impl NetworkService { /// The stream never ends (unless the `NetworkWorker` gets shut down). pub fn event_stream(&self) -> impl Stream { // Note: when transitioning to stable futures, remove the `Error` entirely - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_network_event_stream"); let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::EventStream(tx)); rx } @@ -740,7 +777,7 @@ enum ServiceToWorkerMsg { PutValue(record::Key, Vec), AddKnownAddress(PeerId, Multiaddr), SyncFork(Vec, B::Hash, NumberFor), - EventStream(mpsc::UnboundedSender), + EventStream(TracingUnboundedSender), WriteNotification { message: Vec, engine_id: ConsensusEngineId, @@ -771,11 +808,11 @@ pub struct NetworkWorker { /// The import queue that was passed as initialization. import_queue: Box>, /// Messages from the `NetworkService` and that must be processed. - from_worker: mpsc::UnboundedReceiver>, + from_worker: TracingUnboundedReceiver>, /// Receiver for queries from the light client that must be processed. - light_client_rqs: Option>>, + light_client_rqs: Option>>, /// Senders for events that happen on the network. - event_streams: Vec>, + event_streams: Vec>, /// Prometheus network metrics. metrics: Option, /// The `PeerId`'s of all boot nodes. @@ -784,19 +821,28 @@ pub struct NetworkWorker { struct Metrics { // This list is ordered alphabetically - connections: Gauge, + connections: GaugeVec, + connections_closed_total: CounterVec, import_queue_blocks_submitted: Counter, import_queue_finality_proofs_submitted: Counter, import_queue_justifications_submitted: Counter, + incoming_connections_errors_total: Counter, + incoming_connections_total: Counter, is_major_syncing: Gauge, + issued_light_requests: Counter, kbuckets_num_nodes: Gauge, + listeners_local_addresses: Gauge, + listeners_errors_total: Counter, network_per_sec_bytes: GaugeVec, - notifications_total: CounterVec, + notifications_queues_size: HistogramVec, + notifications_sizes: HistogramVec, num_event_stream_channels: Gauge, opened_notification_streams: GaugeVec, peers_count: Gauge, peerset_num_discovered: Gauge, peerset_num_requested: Gauge, + pending_connections: Gauge, + pending_connections_errors_total: CounterVec, random_kademalia_queries_total: Counter, } @@ -804,8 +850,19 @@ impl Metrics { fn register(registry: &Registry) -> Result { Ok(Self { // This list is ordered alphabetically - connections: register(Gauge::new( - "sub_libp2p_connections", "Number of libp2p connections" + connections: register(GaugeVec::new( + Opts::new( + "sub_libp2p_connections", + "Number of active libp2p connections" + ), + &["direction"] + )?, registry)?, + connections_closed_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_connections_closed_total", + "Total number of connections closed, by reason" + ), + &["reason"] )?, registry)?, import_queue_blocks_submitted: register(Counter::new( "import_queue_blocks_submitted", @@ -819,12 +876,31 @@ impl Metrics { "import_queue_justifications_submitted", "Number of justifications submitted to the import queue.", )?, registry)?, + incoming_connections_errors_total: register(Counter::new( + "sub_libp2p_incoming_connections_handshake_errors_total", + "Total number of incoming connections that have failed during the initial handshake" + )?, registry)?, + incoming_connections_total: register(Counter::new( + "sub_libp2p_incoming_connections_total", + "Total number of incoming connections on the listening sockets" + )?, registry)?, is_major_syncing: register(Gauge::new( "sub_libp2p_is_major_syncing", "Whether the node is performing a major sync or not.", )?, registry)?, + issued_light_requests: register(Counter::new( + "issued_light_requests", + "Number of light client requests that our node has issued.", + )?, registry)?, kbuckets_num_nodes: register(Gauge::new( "sub_libp2p_kbuckets_num_nodes", "Number of nodes in the Kademlia k-buckets" )?, registry)?, + listeners_local_addresses: register(Gauge::new( + "sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on" + )?, registry)?, + listeners_errors_total: register(Counter::new( + "sub_libp2p_listeners_errors_total", + "Total number of non-fatal errors reported by a listener" + )?, registry)?, network_per_sec_bytes: register(GaugeVec::new( Opts::new( "sub_libp2p_network_per_sec_bytes", @@ -832,11 +908,25 @@ impl Metrics { ), &["direction"] )?, registry)?, - notifications_total: register(CounterVec::new( - Opts::new( - "sub_libp2p_notifications_total", - "Number of notification received from all nodes" - ), + notifications_queues_size: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_notifications_queues_size", + "Total size of all the notification queues" + ), + buckets: vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0], + }, + &["protocol"] + )?, registry)?, + notifications_sizes: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_notifications_sizes", + "Sizes of the notifications send to and received from all nodes" + ), + buckets: prometheus_endpoint::exponential_buckets(64.0, 4.0, 8) + .expect("parameters are always valid values; qed"), + }, &["direction", "protocol"] )?, registry)?, num_event_stream_channels: register(Gauge::new( @@ -859,6 +949,17 @@ impl Metrics { peerset_num_requested: register(Gauge::new( "sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to", )?, registry)?, + pending_connections: register(Gauge::new( + "sub_libp2p_pending_connections", + "Number of connections in the process of being established", + )?, registry)?, + pending_connections_errors_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_pending_connections_errors_total", + "Total number of node connection failures" + ), + &["reason"] + )?, registry)?, random_kademalia_queries_total: register(Counter::new( "sub_libp2p_random_kademalia_queries_total", "Number of random Kademlia queries started", )?, registry)?, @@ -874,8 +975,10 @@ impl Metrics { self.opened_notification_streams.with_label_values(&[&engine_id_to_string(&engine_id)]).dec(); }, Event::NotificationsReceived { messages, .. } => { - for (engine_id, _) in messages { - self.notifications_total.with_label_values(&["in", &engine_id_to_string(&engine_id)]).inc(); + for (engine_id, message) in messages { + self.notifications_sizes + .with_label_values(&["in", &engine_id_to_string(&engine_id)]) + .observe(message.len() as f64); } }, _ => {} @@ -897,7 +1000,13 @@ impl Future for NetworkWorker { // Check for new incoming light client requests. if let Some(light_client_rqs) = this.light_client_rqs.as_mut() { while let Poll::Ready(Some(rq)) = light_client_rqs.poll_next_unpin(cx) { - this.network_service.user_protocol_mut().add_light_client_request(rq); + // This can error if there are too many queued requests already. + if this.network_service.light_client_request(rq).is_err() { + log::warn!("Couldn't start light client request: too many pending requests"); + } + if let Some(metrics) = this.metrics.as_ref() { + metrics.issued_light_requests.inc(); + } } } @@ -930,16 +1039,15 @@ impl Future for NetworkWorker { this.event_streams.push(sender), ServiceToWorkerMsg::WriteNotification { message, engine_id, target } => { if let Some(metrics) = this.metrics.as_ref() { - metrics.notifications_total.with_label_values(&["out", &engine_id_to_string(&engine_id)]).inc(); + metrics.notifications_sizes + .with_label_values(&["out", &engine_id_to_string(&engine_id)]) + .observe(message.len() as f64); } this.network_service.user_protocol_mut().write_notification(target, engine_id, message) }, ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => { - let events = this.network_service.user_protocol_mut() + this.network_service .register_notifications_protocol(engine_id, protocol_name); - for event in events { - this.event_streams.retain(|sender| sender.unbounded_send(event.clone()).is_ok()); - } }, ServiceToWorkerMsg::DisconnectPeer(who) => this.network_service.user_protocol_mut().disconnect_peer(&who), @@ -983,25 +1091,51 @@ impl Future for NetworkWorker { metrics.update_with_network_event(&ev); } }, - Poll::Ready(SwarmEvent::Connected(peer_id)) => { + Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, .. }) => { trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); if let Some(metrics) = this.metrics.as_ref() { - metrics.connections.inc(); + match endpoint { + ConnectedPoint::Dialer { .. } => + metrics.connections.with_label_values(&["out"]).inc(), + ConnectedPoint::Listener { .. } => + metrics.connections.with_label_values(&["in"]).inc(), + } + } + }, + Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, .. }) => { + trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?}, {:?})", peer_id, cause); + if let Some(metrics) = this.metrics.as_ref() { + match endpoint { + ConnectedPoint::Dialer { .. } => + metrics.connections.with_label_values(&["out"]).dec(), + ConnectedPoint::Listener { .. } => + metrics.connections.with_label_values(&["in"]).dec(), + } + match cause { + ConnectionError::IO(_) => + metrics.connections_closed_total.with_label_values(&["transport-error"]).inc(), + ConnectionError::ConnectionLimit(_) => + metrics.connections_closed_total.with_label_values(&["limit-reached"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) => + metrics.connections_closed_total.with_label_values(&["protocol-error"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) => + metrics.connections_closed_total.with_label_values(&["keep-alive-timeout"]).inc(), + } } }, - Poll::Ready(SwarmEvent::Disconnected(peer_id)) => { - trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?})", peer_id); + Poll::Ready(SwarmEvent::NewListenAddr(addr)) => { + trace!(target: "sub-libp2p", "Libp2p => NewListenAddr({})", addr); if let Some(metrics) = this.metrics.as_ref() { - metrics.connections.dec(); + metrics.listeners_local_addresses.inc(); } }, - Poll::Ready(SwarmEvent::NewListenAddr(addr)) => - trace!(target: "sub-libp2p", "Libp2p => NewListenAddr({})", addr), - Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => - trace!(target: "sub-libp2p", "Libp2p => ExpiredListenAddr({})", addr), - Poll::Ready(SwarmEvent::UnreachableAddr { peer_id, address, error }) => { - let error = error.to_string(); - + Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => { + trace!(target: "sub-libp2p", "Libp2p => ExpiredListenAddr({})", addr); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_local_addresses.dec(); + } + }, + Poll::Ready(SwarmEvent::UnreachableAddr { peer_id, address, error, .. }) => { trace!( target: "sub-libp2p", "Libp2p => Failed to reach {:?} through {:?}: {}", peer_id, @@ -1009,21 +1143,63 @@ impl Future for NetworkWorker { error, ); - if let Some(peer_id) = peer_id { - if this.boot_node_ids.contains(&peer_id) - && error.contains("Peer ID mismatch") - { + if this.boot_node_ids.contains(&peer_id) { + if let PendingConnectionError::InvalidPeerId = error { error!( - "Connecting to bootnode with peer id `{}` and address `{}` failed \ - because it returned a different peer id!", + "💔 Invalid peer ID from bootnode, expected `{}` at address `{}`.", peer_id, address, ); } } + + if let Some(metrics) = this.metrics.as_ref() { + match error { + PendingConnectionError::InvalidPeerId => + metrics.pending_connections_errors_total.with_label_values(&["invalid-peer-id"]).inc(), + PendingConnectionError::Transport(_) | PendingConnectionError::IO(_) => + metrics.pending_connections_errors_total.with_label_values(&["transport-error"]).inc(), + } + } + } + Poll::Ready(SwarmEvent::Dialing(peer_id)) => + trace!(target: "sub-libp2p", "Libp2p => Dialing({:?})", peer_id), + Poll::Ready(SwarmEvent::IncomingConnection { local_addr, send_back_addr }) => { + trace!(target: "sub-libp2p", "Libp2p => IncomingConnection({},{}))", + local_addr, send_back_addr); + if let Some(metrics) = this.metrics.as_ref() { + metrics.incoming_connections_total.inc(); + } + }, + Poll::Ready(SwarmEvent::IncomingConnectionError { local_addr, send_back_addr, error }) => { + trace!(target: "sub-libp2p", "Libp2p => IncomingConnectionError({},{}): {}", + local_addr, send_back_addr, error); + if let Some(metrics) = this.metrics.as_ref() { + metrics.incoming_connections_errors_total.inc(); + } + }, + Poll::Ready(SwarmEvent::BannedPeer { peer_id, endpoint }) => { + trace!(target: "sub-libp2p", "Libp2p => BannedPeer({}). Connected via {:?}.", + peer_id, endpoint); + if let Some(metrics) = this.metrics.as_ref() { + metrics.incoming_connections_errors_total.inc(); + } + }, + Poll::Ready(SwarmEvent::UnknownPeerUnreachableAddr { address, error }) => + trace!(target: "sub-libp2p", "Libp2p => UnknownPeerUnreachableAddr({}): {}", + address, error), + Poll::Ready(SwarmEvent::ListenerClosed { reason, addresses }) => { + warn!(target: "sub-libp2p", "Libp2p => ListenerClosed: {:?}", reason); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_local_addresses.sub(addresses.len() as u64); + } + }, + Poll::Ready(SwarmEvent::ListenerError { error }) => { + trace!(target: "sub-libp2p", "Libp2p => ListenerError: {}", error); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_errors_total.inc(); + } }, - Poll::Ready(SwarmEvent::StartConnect(peer_id)) => - trace!(target: "sub-libp2p", "Libp2p => StartConnect({:?})", peer_id), }; } @@ -1052,6 +1228,7 @@ impl Future for NetworkWorker { metrics.peers_count.set(num_connected_peers as u64); metrics.peerset_num_discovered.set(this.network_service.user_protocol().num_discovered_peers() as u64); metrics.peerset_num_requested.set(this.network_service.user_protocol().requested_peers().count() as u64); + metrics.pending_connections.set(Swarm::network_info(&this.network_service).num_connections_pending as u64); } Poll::Pending @@ -1090,7 +1267,7 @@ impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { fn justification_imported(&mut self, who: PeerId, hash: &B::Hash, number: NumberFor, success: bool) { self.protocol.user_protocol_mut().justification_import_result(hash.clone(), number, success); if !success { - info!("Invalid justification provided by {} for #{}", who, hash); + info!("💔 Invalid justification provided by {} for #{}", who, hash); self.protocol.user_protocol_mut().disconnect_peer(&who); self.protocol.user_protocol_mut().report_peer(who, ReputationChange::new_fatal("Invalid justification")); } @@ -1110,7 +1287,7 @@ impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { let success = finalization_result.is_ok(); self.protocol.user_protocol_mut().finality_proof_import_result(request_block, finalization_result); if !success { - info!("Invalid finality proof provided by {} for #{}", who, request_block.0); + info!("💔 Invalid finality proof provided by {} for #{}", who, request_block.0); self.protocol.user_protocol_mut().disconnect_peer(&who); self.protocol.user_protocol_mut().report_peer(who, ReputationChange::new_fatal("Invalid finality proof")); } diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index e3d7852dd476ce3a9d0fb490e1fc5c5438df5264..90202008eb37fe99a1a98009bdce923ebe0dfc1c 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -13,10 +13,10 @@ repository = "https://github.com/paritytech/substrate/" sc-network = { version = "0.8.0-alpha.5", path = "../" } log = "0.4.8" parking_lot = "0.10.0" -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.17.0", default-features = false, features = ["libp2p-websocket"] } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } sc-client = { version = "0.8.0-alpha.5", path = "../../" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } @@ -29,3 +29,6 @@ env_logger = "0.7.0" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } tempfile = "3.1.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 8ff06fc5ac9cdfae2d00af0cf2b6fda8ab32e36c..56070f988808112d46473cf550b85916c754b413 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -32,7 +32,7 @@ use sp_blockchain::{ use sc_client_api::{BlockchainEvents, BlockImportNotification, FinalityNotifications, ImportNotifications, FinalityNotification, backend::{TransactionFor, AuxStore, Backend, Finalizer}, BlockBackend}; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sc_client::LongestChain; -use sc_network::config::Roles; +use sc_network::config::Role; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; use sp_consensus::import_queue::{ BasicQueue, BoxJustificationImport, Verifier, BoxFinalityProofImport, @@ -281,7 +281,8 @@ impl Peer { Default::default() }; self.block_import.import_block(import_block, cache).expect("block_import failed"); - self.network.on_block_imported(header, Vec::new(), true); + self.network.on_block_imported(header, true); + self.network.service().announce_block(hash, Vec::new()); at = hash; } @@ -556,17 +557,17 @@ pub trait TestNetFactory: Sized { for i in 0..n { trace!(target: "test_network", "Adding peer {}", i); - net.add_full_peer(&config); + net.add_full_peer(); } net } - fn add_full_peer(&mut self, config: &ProtocolConfig) { - self.add_full_peer_with_states(config, None) + fn add_full_peer(&mut self) { + self.add_full_peer_with_states(None) } /// Add a full peer. - fn add_full_peer_with_states(&mut self, config: &ProtocolConfig, keep_blocks: Option) { + fn add_full_peer_with_states(&mut self, keep_blocks: Option) { let test_client_builder = match keep_blocks { Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks), None => TestClientBuilder::with_default_backend(), @@ -585,7 +586,7 @@ pub trait TestNetFactory: Sized { let verifier = self.make_verifier( PeersClient::Full(client.clone(), backend.clone()), - config, + &Default::default(), &data, ); let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); @@ -599,14 +600,19 @@ pub trait TestNetFactory: Sized { let listen_addr = build_multiaddr![Memory(rand::random::())]; + let mut network_config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + &std::env::current_dir().expect("current directory must exist"), + ); + network_config.transport = TransportConfig::MemoryOnly; + network_config.listen_addresses = vec![listen_addr.clone()]; + let network = NetworkWorker::new(sc_network::config::Params { - roles: config.roles, + role: Role::Full, executor: None, - network_config: NetworkConfiguration { - listen_addresses: vec![listen_addr.clone()], - transport: TransportConfig::MemoryOnly, - ..NetworkConfiguration::default() - }, + network_config, chain: client.clone(), finality_proof_provider: self.make_finality_proof_provider( PeersClient::Full(client.clone(), backend.clone()), @@ -643,10 +649,7 @@ pub trait TestNetFactory: Sized { } /// Add a light peer. - fn add_light_peer(&mut self, config: &ProtocolConfig) { - let mut config = config.clone(); - config.roles = Roles::LIGHT; - + fn add_light_peer(&mut self) { let (c, backend) = substrate_test_runtime_client::new_light(); let client = Arc::new(c); let ( @@ -659,7 +662,7 @@ pub trait TestNetFactory: Sized { let verifier = self.make_verifier( PeersClient::Light(client.clone(), backend.clone()), - &config, + &Default::default(), &data, ); let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); @@ -673,14 +676,19 @@ pub trait TestNetFactory: Sized { let listen_addr = build_multiaddr![Memory(rand::random::())]; + let mut network_config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + &std::env::current_dir().expect("current directory must exist"), + ); + network_config.transport = TransportConfig::MemoryOnly; + network_config.listen_addresses = vec![listen_addr.clone()]; + let network = NetworkWorker::new(sc_network::config::Params { - roles: config.roles, + role: Role::Light, executor: None, - network_config: NetworkConfiguration { - listen_addresses: vec![listen_addr.clone()], - transport: TransportConfig::MemoryOnly, - ..NetworkConfiguration::default() - }, + network_config, chain: client.clone(), finality_proof_provider: self.make_finality_proof_provider( PeersClient::Light(client.clone(), backend.clone()) @@ -785,9 +793,9 @@ pub trait TestNetFactory: Sized { while let Poll::Ready(Some(notification)) = peer.imported_blocks_stream.as_mut().poll_next(cx) { peer.network.on_block_imported( notification.header, - Vec::new(), true, ); + peer.network.service().announce_block(notification.hash, Vec::new()); } // We poll `finality_notification_stream`, but we only take the last event. diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 5453747220817da66ebffa4b4803b236f07a5187..8acf265e9189288758471f69db6873b0418a3edd 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_network::config::Roles; use sp_consensus::BlockOrigin; use std::time::Duration; use futures::executor::block_on; @@ -372,10 +371,8 @@ fn blocks_are_not_announced_by_light_nodes() { // full peer0 is connected to light peer // light peer1 is connected to full peer2 - let mut light_config = ProtocolConfig::default(); - light_config.roles = Roles::LIGHT; - net.add_full_peer(&ProtocolConfig::default()); - net.add_light_peer(&light_config); + net.add_full_peer(); + net.add_light_peer(); // Sync between 0 and 1. net.peer(0).push_blocks(1, false); @@ -384,7 +381,7 @@ fn blocks_are_not_announced_by_light_nodes() { assert_eq!(net.peer(1).client.info().best_number, 1); // Add another node and remove node 0. - net.add_full_peer(&ProtocolConfig::default()); + net.add_full_peer(); net.peers.remove(0); // Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together. @@ -465,7 +462,7 @@ fn can_not_sync_from_light_peer() { // given the network with 1 full nodes (#0) and 1 light node (#1) let mut net = TestNet::new(1); - net.add_light_peer(&Default::default()); + net.add_light_peer(); // generate some blocks on #0 net.peer(0).push_blocks(1, false); @@ -481,7 +478,7 @@ fn can_not_sync_from_light_peer() { assert_eq!(light_info.best_hash, full0_info.best_hash); // add new full client (#2) && remove #0 - net.add_full_peer(&Default::default()); + net.add_full_peer(); net.peers.remove(0); // ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds @@ -511,7 +508,7 @@ fn light_peer_imports_header_from_announce() { // given the network with 1 full nodes (#0) and 1 light node (#1) let mut net = TestNet::new(1); - net.add_light_peer(&Default::default()); + net.add_light_peer(); // let them connect to each other net.block_until_sync(); @@ -583,9 +580,8 @@ fn can_sync_explicit_forks() { fn syncs_header_only_forks() { let _ = ::env_logger::try_init(); let mut net = TestNet::new(0); - let config = ProtocolConfig::default(); - net.add_full_peer_with_states(&config, None); - net.add_full_peer_with_states(&config, Some(3)); + net.add_full_peer_with_states(None); + net.add_full_peer_with_states(Some(3)); net.peer(0).push_blocks(2, false); net.peer(1).push_blocks(2, false); @@ -694,3 +690,32 @@ fn imports_stale_once() { assert_eq!(net.peer(1).num_processed_blocks(), 2); } +#[test] +fn can_sync_to_peers_with_wrong_common_block() { + let _ = ::env_logger::try_init(); + let mut net = TestNet::new(2); + + net.peer(0).push_blocks(2, true); + net.peer(1).push_blocks(2, true); + let fork_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 2, false); + net.peer(1).push_blocks_at(BlockId::Number(0), 2, false); + // wait for connection + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { + Poll::Pending + } else { + Poll::Ready(()) + } + })); + + // both peers re-org to the same fork without notifying each other + net.peer(0).client().finalize_block(BlockId::Hash(fork_hash), Some(Vec::new()), true).unwrap(); + net.peer(1).client().finalize_block(BlockId::Hash(fork_hash), Some(Vec::new()), true).unwrap(); + let final_hash = net.peer(0).push_blocks(1, false); + + net.block_until_sync(); + + assert!(net.peer(1).client().header(&BlockId::Hash(final_hash)).unwrap().is_some()); +} + diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 9dac18e946ebcc699633165cb8e367baefc7f74a..e7292439e82eb0d4ad27e9f64c897c764b2f18f4 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -13,17 +13,18 @@ bytes = "0.5" sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } fnv = "1.0.6" -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" sp-offchain = { version = "2.0.0-alpha.5", path = "../../primitives/offchain" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } rand = "0.7.2" sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sc-network = { version = "0.8.0-alpha.5", path = "../network" } sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } @@ -42,3 +43,6 @@ tokio = "0.2" [features] default = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index 7923a767f11f0577828168592360894e2a4e53ab..a64fe0389706c1098eda9f1c2fbd2294843d7597 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -32,11 +32,12 @@ use futures::{prelude::*, future, channel::mpsc}; use log::error; use sp_core::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError}; use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll}; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; /// Creates a pair of [`HttpApi`] and [`HttpWorker`]. pub fn http() -> (HttpApi, HttpWorker) { - let (to_worker, from_api) = mpsc::unbounded(); - let (to_api, from_worker) = mpsc::unbounded(); + let (to_worker, from_api) = tracing_unbounded("mpsc_ocw_to_worker"); + let (to_api, from_worker) = tracing_unbounded("mpsc_ocw_to_api"); let api = HttpApi { to_worker, @@ -63,10 +64,10 @@ pub fn http() -> (HttpApi, HttpWorker) { /// to offchain workers. pub struct HttpApi { /// Used to sends messages to the worker. - to_worker: mpsc::UnboundedSender, + to_worker: TracingUnboundedSender, /// Used to receive messages from the worker. /// We use a `Fuse` in order to have an extra protection against panicking. - from_worker: stream::Fuse>, + from_worker: stream::Fuse>, /// Id to assign to the next HTTP request that is started. next_id: HttpRequestId, /// List of HTTP requests in preparation or in progress. @@ -546,9 +547,9 @@ enum WorkerToApi { /// Must be continuously polled for the [`HttpApi`] to properly work. pub struct HttpWorker { /// Used to sends messages to the `HttpApi`. - to_api: mpsc::UnboundedSender, + to_api: TracingUnboundedSender, /// Used to receive messages from the `HttpApi`. - from_api: mpsc::UnboundedReceiver, + from_api: TracingUnboundedReceiver, /// The engine that runs HTTP requests. http_client: hyper::Client, hyper::Body>, /// HTTP requests that are being worked on by the engine. diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 27a7f508459ba1bbc3cc4ba0eef0ed2a7c5f9770..94850e3fd3461c3cf79ecd48e905051991324941 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -191,7 +191,8 @@ mod tests { at: &BlockId, extrinsic: ::Extrinsic, ) -> Result<(), ()> { - futures::executor::block_on(self.0.submit_one(&at, extrinsic)) + let source = sp_transaction_pool::TransactionSource::Local; + futures::executor::block_on(self.0.submit_one(&at, source, extrinsic)) .map(|_| ()) .map_err(|_| ()) } diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index ea3a6b6974819bfc1677427635c0fb1b85ac4699..e026c6063a01fbd9e6716d0a3fe3b59cf23302eb 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -11,11 +11,15 @@ documentation = "https://docs.rs/sc-peerset" [dependencies] -futures = "0.3.1" -libp2p = { version = "0.16.2", default-features = false } +futures = "0.3.4" +libp2p = { version = "0.17.0", default-features = false } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils"} log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" [dev-dependencies] rand = "0.7.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 87ed2336aea9c65da74aafbc53216ce0a6ffaff7..9376e9594b4b702dc3746661196065e4851c78b9 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -20,11 +20,12 @@ mod peersstate; use std::{collections::{HashSet, HashMap}, collections::VecDeque}; -use futures::{prelude::*, channel::mpsc}; +use futures::prelude::*; use log::{debug, error, trace}; use serde_json::json; use std::{pin::Pin, task::{Context, Poll}, time::Duration}; use wasm_timer::Instant; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; pub use libp2p::PeerId; @@ -73,7 +74,7 @@ impl ReputationChange { /// Shared handle to the peer set manager (PSM). Distributed around the code. #[derive(Debug, Clone)] pub struct PeersetHandle { - tx: mpsc::UnboundedSender, + tx: TracingUnboundedSender, } impl PeersetHandle { @@ -163,14 +164,14 @@ pub struct PeersetConfig { /// > otherwise it will not be able to connect to them. pub bootnodes: Vec, - /// If true, we only accept reserved nodes. + /// If true, we only accept nodes in [`PeersetConfig::priority_groups`]. pub reserved_only: bool, - /// List of nodes that we should always be connected to. + /// Lists of nodes we should always be connected to. /// /// > **Note**: Keep in mind that the networking has to know an address for these nodes, /// > otherwise it will not be able to connect to them. - pub reserved_nodes: Vec, + pub priority_groups: Vec<(String, HashSet)>, } /// Side of the peer set manager owned by the network. In other words, the "receiving" side. @@ -183,9 +184,9 @@ pub struct Peerset { /// If true, we only accept reserved nodes. reserved_only: bool, /// Receiver for messages from the `PeersetHandle` and from `tx`. - rx: mpsc::UnboundedReceiver, + rx: TracingUnboundedReceiver, /// Sending side of `rx`. - tx: mpsc::UnboundedSender, + tx: TracingUnboundedSender, /// Queue of messages to be emitted when the `Peerset` is polled. message_queue: VecDeque, /// When the `Peerset` was created. @@ -197,7 +198,7 @@ pub struct Peerset { impl Peerset { /// Builds a new peerset from the given configuration. pub fn from_config(config: PeersetConfig) -> (Peerset, PeersetHandle) { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_peerset_messages"); let handle = PeersetHandle { tx: tx.clone(), @@ -215,7 +216,10 @@ impl Peerset { latest_time_update: now, }; - peerset.data.set_priority_group(RESERVED_NODES, config.reserved_nodes.into_iter().collect()); + for (group, nodes) in config.priority_groups { + peerset.data.set_priority_group(&group, nodes); + } + for peer_id in config.bootnodes { if let peersstate::Peer::Unknown(entry) = peerset.data.peer(&peer_id) { entry.discover(); @@ -597,7 +601,7 @@ mod tests { out_peers: 2, bootnodes: vec![bootnode], reserved_only: true, - reserved_nodes: Vec::new(), + priority_groups: Vec::new(), }; let (peerset, handle) = Peerset::from_config(config); @@ -625,7 +629,7 @@ mod tests { out_peers: 1, bootnodes: vec![bootnode.clone()], reserved_only: false, - reserved_nodes: Vec::new(), + priority_groups: Vec::new(), }; let (mut peerset, _handle) = Peerset::from_config(config); @@ -652,7 +656,7 @@ mod tests { out_peers: 2, bootnodes: vec![bootnode.clone()], reserved_only: false, - reserved_nodes: vec![], + priority_groups: vec![], }; let (mut peerset, _handle) = Peerset::from_config(config); @@ -673,7 +677,7 @@ mod tests { out_peers: 25, bootnodes: vec![], reserved_only: false, - reserved_nodes: vec![], + priority_groups: vec![], }); // We ban a node by setting its reputation under the threshold. diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index c2b0b44a3a95d43f30210efc73563e4e0ae520a3..44477cec6589dbf2b6ce5922ae4ac3217fe9e973 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -43,12 +43,15 @@ fn test_once() { known_nodes.insert(id.clone()); id }).collect(), - reserved_nodes: (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| { - let id = PeerId::random(); - known_nodes.insert(id.clone()); - reserved_nodes.insert(id.clone()); - id - }).collect(), + priority_groups: { + let list = (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| { + let id = PeerId::random(); + known_nodes.insert(id.clone()); + reserved_nodes.insert(id.clone()); + id + }).collect(); + vec![("reserved".to_owned(), list)] + }, reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 7d41c4f61cc68f00a5790e5cd210e85bb14509a0..c5043b5b0782b72d00e0f9940d0f6e87f9963c9c 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -9,11 +9,11 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC interfaces." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" +jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" jsonrpc-pubsub = "14.0.3" log = "0.4.8" @@ -25,3 +25,6 @@ serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } sp-rpc = { version = "2.0.0-alpha.5", path = "../../primitives/rpc" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/rpc-api/src/state/mod.rs b/client/rpc-api/src/state/mod.rs index b2cf8ce909b20fead05a3be7bdb9d0d93d6fade1..d29e46a4b5637cf85d0745d59a73b0ad69afde7c 100644 --- a/client/rpc-api/src/state/mod.rs +++ b/client/rpc-api/src/state/mod.rs @@ -136,6 +136,14 @@ pub trait StateApi { hash: Option ) -> FutureResult>>; + /// Query storage entries (by key) starting at block hash given as the second parameter. + #[rpc(name = "state_queryStorageAt")] + fn query_storage_at( + &self, + keys: Vec, + at: Option, + ) -> FutureResult>>; + /// New runtime version subscription #[pubsub( subscription = "state_runtimeVersion", diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs index 572136aeb6c40f515276c5c727798602b09559fd..80718cf487c8999df8f14c26a40673d41c57723a 100644 --- a/client/rpc-api/src/system/helpers.rs +++ b/client/rpc-api/src/system/helpers.rs @@ -83,8 +83,8 @@ pub enum NodeRole { LightClient, /// The node is an authority Authority, - /// An unknown role with a bit number - UnknownRole(u8) + /// The node is a sentry + Sentry, } #[cfg(test)] diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index 5e0feb64c8e9697a319c492804ca3ed7877c28e5..c834d7dbf7396b89a6a72ccaaf7feca3059e7507 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -19,3 +19,6 @@ sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } [target.'cfg(not(target_os = "unknown"))'.dependencies] http = { package = "jsonrpc-http-server", version = "14.0.3" } ws = { package = "jsonrpc-ws-server", version = "14.0.3" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index cea58e31bf2f2461db92e01f0f8654b50a38fb8b..7092959b83e9c450984b14b2394dd738ea440a2f 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -13,7 +13,7 @@ sc-rpc-api = { version = "0.8.0-alpha.5", path = "../rpc-api" } sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sc-client = { version = "0.8.0-alpha.5", path = "../" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } futures = { version = "0.3.1", features = ["compat"] } jsonrpc-pubsub = "14.0.3" log = "0.4.8" @@ -24,6 +24,7 @@ serde_json = "1.0.41" sp-session = { version = "2.0.0-alpha.5", path = "../../primitives/session" } sp-offchain = { version = "2.0.0-alpha.5", path = "../../primitives/offchain" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-rpc = { version = "2.0.0-alpha.5", path = "../../primitives/rpc" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } @@ -37,8 +38,11 @@ parking_lot = "0.10.0" [dev-dependencies] assert_matches = "1.3.0" futures01 = { package = "futures", version = "0.1.29" } -sc-network = { version = "0.8.0-alpha.4", path = "../network" } -sp-io = { version = "2.0.0-alpha.4", path = "../../primitives/io" } +sc-network = { version = "0.8.0-alpha.5", path = "../network" } +sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.1.22" sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../transaction-pool" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 80a3a4349ed82f5fa2ebe04d96cf24d28e18ae5e..a3f23e8e1437a146932410d08ce6835b40907a6d 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -37,7 +37,7 @@ use sp_core::{Bytes, traits::BareCryptoStorePtr}; use sp_api::ProvideRuntimeApi; use sp_runtime::generic; use sp_transaction_pool::{ - TransactionPool, InPoolTransaction, TransactionStatus, + TransactionPool, InPoolTransaction, TransactionStatus, TransactionSource, BlockHash, TxHash, TransactionFor, error::IntoPoolError, }; use sp_session::SessionKeys; @@ -75,6 +75,14 @@ impl Author { } } + +/// Currently we treat all RPC transactions as externals. +/// +/// Possibly in the future we could allow opt-in for special treatment +/// of such transactions, so that the block authors can inject +/// some unique transactions via RPC and have them included in the pool. +const TX_SOURCE: TransactionSource = TransactionSource::External; + impl AuthorApi, BlockHash

> for Author where P: TransactionPool + Sync + Send + 'static, @@ -127,7 +135,7 @@ impl AuthorApi, BlockHash

> for Author }; let best_block_hash = self.client.info().best_hash; Box::new(self.pool - .submit_one(&generic::BlockId::hash(best_block_hash), xt) + .submit_one(&generic::BlockId::hash(best_block_hash), TX_SOURCE, xt) .compat() .map_err(|e| e.into_pool_error() .map(Into::into) @@ -173,7 +181,7 @@ impl AuthorApi, BlockHash

> for Author .map_err(error::Error::from)?; Ok( self.pool - .submit_and_watch(&generic::BlockId::hash(best_block_hash), dxt) + .submit_and_watch(&generic::BlockId::hash(best_block_hash), TX_SOURCE, dxt) .map_err(|e| e.into_pool_error() .map(error::Error::from) .unwrap_or_else(|e| error::Error::Verification(Box::new(e)).into()) diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index 3093cd9d3b759b08d887eb10165e26f697a9a073..8b956c23a5e660cd685c9d6381395068decb4ebc 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -21,7 +21,8 @@ use assert_matches::assert_matches; use codec::Encode; use sp_core::{ H256, blake2_256, hexdisplay::HexDisplay, testing::{ED25519, SR25519, KeyStore}, - traits::BareCryptoStorePtr, ed25519, crypto::{Pair, Public}, + traits::BareCryptoStorePtr, ed25519, sr25519, + crypto::{CryptoTypePublicPair, Pair, Public}, }; use rpc::futures::Stream as _; use substrate_test_runtime_client::{ @@ -173,7 +174,7 @@ fn should_return_pending_extrinsics() { let ex = uxt(AccountKeyring::Alice, 0); AuthorApi::submit_extrinsic(&p, ex.encode().into()).wait().unwrap(); - assert_matches!( + assert_matches!( p.pending_extrinsics(), Ok(ref expected) if *expected == vec![Bytes(ex.encode())] ); @@ -199,7 +200,7 @@ fn should_remove_extrinsics() { hash::ExtrinsicOrHash::Extrinsic(ex1.encode().into()), ]).unwrap(); - assert_eq!(removed.len(), 3); + assert_eq!(removed.len(), 3); } #[test] @@ -215,10 +216,9 @@ fn should_insert_key() { key_pair.public().0.to_vec().into(), ).expect("Insert key"); - let store_key_pair = setup.keystore.read() - .ed25519_key_pair(ED25519, &key_pair.public()).expect("Key exists in store"); + let public_keys = setup.keystore.read().keys(ED25519).unwrap(); - assert_eq!(key_pair.public(), store_key_pair.public()); + assert!(public_keys.contains(&CryptoTypePublicPair(ed25519::CRYPTO_ID, key_pair.public().to_raw_vec()))); } #[test] @@ -231,18 +231,11 @@ fn should_rotate_keys() { let session_keys = SessionKeys::decode(&mut &new_public_keys[..]) .expect("SessionKeys decode successfully"); - let ed25519_key_pair = setup.keystore.read().ed25519_key_pair( - ED25519, - &session_keys.ed25519.clone().into(), - ).expect("ed25519 key exists in store"); - - let sr25519_key_pair = setup.keystore.read().sr25519_key_pair( - SR25519, - &session_keys.sr25519.clone().into(), - ).expect("sr25519 key exists in store"); + let ed25519_public_keys = setup.keystore.read().keys(ED25519).unwrap(); + let sr25519_public_keys = setup.keystore.read().keys(SR25519).unwrap(); - assert_eq!(session_keys.ed25519, ed25519_key_pair.public().into()); - assert_eq!(session_keys.sr25519, sr25519_key_pair.public().into()); + assert!(ed25519_public_keys.contains(&CryptoTypePublicPair(ed25519::CRYPTO_ID, session_keys.ed25519.to_raw_vec()))); + assert!(sr25519_public_keys.contains(&CryptoTypePublicPair(sr25519::CRYPTO_ID, session_keys.sr25519.to_raw_vec()))); } #[test] diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 82568866ee3ba4df6686d9201783f7cf8cfa673e..2747405c04fcac0c7534ea802bb5e1f08a7e1911 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -163,6 +163,13 @@ pub trait StateBackend: Send + Sync + 'static keys: Vec, ) -> FutureResult>>; + /// Query storage entries (by key) starting at block hash given as the second parameter. + fn query_storage_at( + &self, + keys: Vec, + at: Option + ) -> FutureResult>>; + /// New runtime version subscription fn subscribe_runtime_version( &self, @@ -357,6 +364,14 @@ impl StateApi for State self.backend.query_storage(from, to, keys) } + fn query_storage_at( + &self, + keys: Vec, + at: Option + ) -> FutureResult>> { + self.backend.query_storage_at(keys, at) + } + fn subscribe_storage( &self, meta: Self::Metadata, diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index b7589d2aefecafc9daa41e2122461c21f53c93d2..bf80820543102483ae5c41a301ad1ab0d322890d 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -33,7 +33,7 @@ use sp_core::{ }; use sp_version::RuntimeVersion; use sp_runtime::{ - generic::BlockId, traits::{Block as BlockT, NumberFor, SaturatedConversion}, + generic::BlockId, traits::{Block as BlockT, NumberFor, SaturatedConversion, CheckedSub}, }; use sp_api::{Metadata, ProvideRuntimeApi, CallApiAt}; @@ -94,8 +94,8 @@ impl FullState let from_meta = self.client.header_metadata(from).map_err(invalid_block_err)?; let to_meta = self.client.header_metadata(to).map_err(invalid_block_err)?; - if from_meta.number >= to_meta.number { - return Err(invalid_block_range(&from_meta, &to_meta, "from number >= to number".to_owned())) + if from_meta.number > to_meta.number { + return Err(invalid_block_range(&from_meta, &to_meta, "from number > to number".to_owned())) } // check if we can get from `to` to `from` by going through parent_hashes. @@ -122,7 +122,10 @@ impl FullState .max_key_changes_range(from_number, BlockId::Hash(to_meta.hash)) .map_err(client_err)?; let filtered_range_begin = changes_trie_range - .map(|(begin, _)| (begin - from_number).saturated_into::()); + .and_then(|(begin, _)| { + // avoids a corner case where begin < from_number (happens when querying genesis) + begin.checked_sub(&from_number).map(|x| x.saturated_into::()) + }); let (unfiltered_range, filtered_range) = split_range(hashes.len(), filtered_range_begin); Ok(QueryStorageRange { @@ -398,6 +401,15 @@ impl StateBackend for FullState, + at: Option + ) -> FutureResult>> { + let at = at.unwrap_or_else(|| self.client.info().best_hash); + self.query_storage(at, Some(at), keys) + } + fn subscribe_runtime_version( &self, _meta: crate::metadata::Metadata, diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs index 59c0f2183cf898f0677ec9e39b843378244a7fda..092419ad0129e6f4109a19d3f916dc0eb24f5738 100644 --- a/client/rpc/src/state/state_light.rs +++ b/client/rpc/src/state/state_light.rs @@ -331,6 +331,14 @@ impl StateBackend for LightState, + _at: Option + ) -> FutureResult>> { + Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) + } + fn subscribe_storage( &self, _meta: crate::metadata::Metadata, diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 75ce4ed9c3c4d7e89a4bfb8d691e9a06d2654c7c..4a9b701959c8c292a3b4dba25d89762fedfd8f62 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -30,6 +30,7 @@ use substrate_test_runtime_client::{ sp_consensus::BlockOrigin, runtime, }; +use sp_runtime::generic::BlockId; const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id"); @@ -212,7 +213,7 @@ fn should_send_initial_storage_changes_and_notifications() { #[test] fn should_query_storage() { - fn run_tests(mut client: Arc) { + fn run_tests(mut client: Arc, has_changes_trie_config: bool) { let core = tokio::runtime::Runtime::new().unwrap(); let api = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); @@ -237,6 +238,13 @@ fn should_query_storage() { let block2_hash = add_block(1); let genesis_hash = client.genesis_hash(); + if has_changes_trie_config { + assert_eq!( + client.max_key_changes_range(1, BlockId::Hash(block1_hash)).unwrap(), + Some((0, BlockId::Hash(block1_hash))), + ); + } + let mut expected = vec![ StorageChangeSet { block: genesis_hash, @@ -306,7 +314,7 @@ fn should_query_storage() { Err(Error::InvalidBlockRange { from: format!("1 ({:?})", block1_hash), to: format!("0 ({:?})", genesis_hash), - details: "from number >= to number".to_owned(), + details: "from number > to number".to_owned(), }).map_err(|e| e.to_string()) ); @@ -376,12 +384,39 @@ fn should_query_storage() { details: format!("UnknownBlock: header not found in db: {}", random_hash1), }).map_err(|e| e.to_string()), ); + + // single block range + let result = api.query_storage_at( + keys.clone(), + Some(block1_hash), + ); + + assert_eq!( + result.wait().unwrap(), + vec![ + StorageChangeSet { + block: block1_hash, + changes: vec![ + (StorageKey(vec![1_u8]), None), + (StorageKey(vec![2_u8]), Some(StorageData(vec![2_u8]))), + (StorageKey(vec![3_u8]), Some(StorageData(vec![3_u8]))), + (StorageKey(vec![4_u8]), None), + (StorageKey(vec![5_u8]), Some(StorageData(vec![0_u8]))), + ] + } + ] + ); } - run_tests(Arc::new(substrate_test_runtime_client::new())); - run_tests(Arc::new(TestClientBuilder::new() - .changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2))) - .build())); + run_tests(Arc::new(substrate_test_runtime_client::new()), false); + run_tests( + Arc::new( + TestClientBuilder::new() + .changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2))) + .build(), + ), + true, + ); } #[test] @@ -403,7 +438,7 @@ fn should_return_runtime_version() { let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",2],\ - [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",1],[\"0x40fe3ad401f8959a\",4],\ + [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",2],[\"0x40fe3ad401f8959a\",4],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",1],\ [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]]}"; diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index 3a9ed9f2dcc58a2ad8b3e4064014883085443199..9565992cf44c526ecf235185be4cfd3243fe1020 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -20,8 +20,9 @@ mod tests; use futures::{future::BoxFuture, FutureExt, TryFutureExt}; -use futures::{channel::{mpsc, oneshot}, compat::Compat}; +use futures::{channel::oneshot, compat::Compat}; use sc_rpc_api::Receiver; +use sp_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::{self, Header as HeaderT}; use self::error::Result; @@ -33,7 +34,7 @@ pub use self::gen_client::Client as SystemClient; /// System API implementation pub struct System { info: SystemInfo, - send_back: mpsc::UnboundedSender>, + send_back: TracingUnboundedSender>, } /// Request to be processed. @@ -59,7 +60,7 @@ impl System { /// reading from that channel and answering the requests. pub fn new( info: SystemInfo, - send_back: mpsc::UnboundedSender>, + send_back: TracingUnboundedSender>, ) -> Self { System { info, diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index 4487566e44cfd9fc18b41112d9ec66555c728196..d45894743c104013d4eee67b85cebdb9b1bc3360 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -17,7 +17,7 @@ use super::*; use sc_network::{self, PeerId}; -use sc_network::config::Roles; +use sc_network::config::Role; use substrate_test_runtime_client::runtime::Block; use assert_matches::assert_matches; use futures::{prelude::*, channel::mpsc}; @@ -60,7 +60,7 @@ fn api>>(sync: T) -> System { for _peer in 0..status.peers { peers.push(PeerInfo { peer_id: status.peer_id.to_base58(), - roles: format!("{:?}", Roles::FULL), + roles: format!("{}", Role::Full), protocol_version: 1, best_hash: Default::default(), best_number: 1, diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index eb9e3dc2a655215e318d1d41a1daedee7964d984..634d774a313d20593485a80a2c780911c2aea501 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -20,7 +20,7 @@ wasmtime = [ [dependencies] derive_more = "0.99.2" futures01 = { package = "futures", version = "0.1.29" } -futures = "0.3.1" +futures = "0.3.4" futures-diagnose = "1.0" parking_lot = "0.10.0" lazy_static = "1.4.0" @@ -32,10 +32,10 @@ exit-future = "0.2.0" serde = "1.0.101" serde_json = "1.0.41" sysinfo = "0.12.0" -target_info = "0.1.0" sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-session = { version = "2.0.0-alpha.5", path = "../../primitives/session" } @@ -47,7 +47,7 @@ sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sc-client = { version = "0.8.0-alpha.5", path = "../" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } sc-client-db = { version = "0.8.0-alpha.5", path = "../db" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../transaction-pool" } sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } @@ -61,8 +61,19 @@ sc-tracing = { version = "2.0.0-alpha.5", path = "../tracing" } tracing = "0.1.10" parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } + +[target.'cfg(any(unix, windows))'.dependencies] +netstat2 = "0.8.1" + +[target.'cfg(target_os = "linux")'.dependencies] +procfs = '0.7.8' + + [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/babe" } grandpa = { version = "0.8.0-alpha.5", package = "sc-finality-grandpa", path = "../finality-grandpa" } grandpa-primitives = { version = "2.0.0-alpha.5", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 4363e204c07a8350a96c474ca2f7e6906411e911..0586a61a89715cd39cbb00849fdc1678a76eb86f 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -18,6 +18,7 @@ use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL use crate::{TaskManagerBuilder, start_rpc_servers, build_network_future, TransactionPoolAdapter}; use crate::status_sinks; use crate::config::{Configuration, DatabaseConfig, KeystoreConfig, PrometheusConfig}; +use crate::metrics::MetricsService; use sc_client_api::{ self, BlockchainEvents, @@ -25,22 +26,22 @@ use sc_client_api::{ execution_extensions::ExtensionsFactory, ExecutorProvider, CallExecutor }; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sc_client::Client; use sc_chain_spec::get_extension; use sp_consensus::import_queue::ImportQueue; use futures::{ Future, FutureExt, StreamExt, - channel::mpsc, future::ready, }; use sc_keystore::{Store as Keystore}; use log::{info, warn, error}; -use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; +use sc_network::config::{Role, FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; use sc_network::{NetworkService, NetworkStateInfo}; use parking_lot::{Mutex, RwLock}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, NumberFor, SaturatedConversion, HashFor, UniqueSaturatedInto, + Block as BlockT, NumberFor, SaturatedConversion, HashFor, }; use sp_api::ProvideRuntimeApi; use sc_executor::{NativeExecutor, NativeExecutionDispatch}; @@ -49,56 +50,9 @@ use std::{ marker::PhantomData, sync::Arc, pin::Pin }; use wasm_timer::SystemTime; -use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sp_blockchain; -use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; - -struct ServiceMetrics { - block_height_number: GaugeVec, - ready_transactions_number: Gauge, - memory_usage_bytes: Gauge, - cpu_usage_percentage: Gauge, - network_per_sec_bytes: GaugeVec, - database_cache: Gauge, - state_cache: Gauge, - state_db: GaugeVec, -} - -impl ServiceMetrics { - fn register(registry: &Registry) -> Result { - Ok(Self { - block_height_number: register(GaugeVec::new( - Opts::new("block_height_number", "Height of the chain"), - &["status"] - )?, registry)?, - ready_transactions_number: register(Gauge::new( - "ready_transactions_number", "Number of transactions in the ready queue", - )?, registry)?, - memory_usage_bytes: register(Gauge::new( - "memory_usage_bytes", "Node memory usage", - )?, registry)?, - cpu_usage_percentage: register(Gauge::new( - "cpu_usage_percentage", "Node CPU usage", - )?, registry)?, - network_per_sec_bytes: register(GaugeVec::new( - Opts::new("network_per_sec_bytes", "Networking bytes per second"), - &["direction"] - )?, registry)?, - database_cache: register(Gauge::new( - "database_cache_bytes", "RocksDB cache size in bytes", - )?, registry)?, - state_cache: register(Gauge::new( - "state_cache_bytes", "State cache size in bytes", - )?, registry)?, - state_db: register(GaugeVec::new( - Opts::new("state_db_cache_bytes", "State DB cache in bytes"), - &["subtype"] - )?, registry)?, - }) - } -} pub type BackgroundTask = Pin + Send>>; @@ -214,7 +168,6 @@ fn new_full_parts( password.clone() )?, KeystoreConfig::InMemory => Keystore::new_in_memory(), - KeystoreConfig::None => return Err("No keystore config provided!".into()), }; let tasks_builder = TaskManagerBuilder::new(); @@ -225,7 +178,7 @@ fn new_full_parts( config.max_runtime_instances, ); - let chain_spec = config.expect_chain_spec(); + let chain_spec = &config.chain_spec; let fork_blocks = get_extension::>(chain_spec.extensions()) .cloned() .unwrap_or_default(); @@ -240,11 +193,11 @@ fn new_full_parts( state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), pruning: config.pruning.clone(), - source: match config.expect_database() { + source: match &config.database { DatabaseConfig::Path { path, cache_size } => sc_client_db::DatabaseSettingsSrc::Path { path: path.clone(), - cache_size: cache_size.clone().map(|u| u as usize), + cache_size: *cache_size, }, DatabaseConfig::Custom(db) => sc_client_db::DatabaseSettingsSrc::Custom(db.clone()), @@ -259,7 +212,7 @@ fn new_full_parts( sc_client_db::new_client( db_config, executor, - config.expect_chain_spec().as_storage_builder(), + chain_spec.as_storage_builder(), fork_blocks, bad_blocks, extensions, @@ -335,7 +288,6 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { password.clone() )?, KeystoreConfig::InMemory => Keystore::new_in_memory(), - KeystoreConfig::None => return Err("No keystore config provided!".into()), }; let executor = NativeExecutor::::new( @@ -350,11 +302,11 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), pruning: config.pruning.clone(), - source: match config.expect_database() { + source: match &config.database { DatabaseConfig::Path { path, cache_size } => sc_client_db::DatabaseSettingsSrc::Path { path: path.clone(), - cache_size: cache_size.clone().map(|u| u as usize), + cache_size: *cache_size, }, DatabaseConfig::Custom(db) => sc_client_db::DatabaseSettingsSrc::Custom(db.clone()), @@ -375,7 +327,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { let remote_blockchain = backend.remote_blockchain(); let client = Arc::new(sc_client::light::new_light( backend.clone(), - config.expect_chain_spec().as_storage_builder(), + config.chain_spec.as_storage_builder(), executor, Box::new(tasks_builder.spawn_handle()), config.prometheus_config.as_ref().map(|config| config.registry.clone()), @@ -820,14 +772,14 @@ ServiceBuilder< )?; // A side-channel for essential tasks to communicate shutdown. - let (essential_failed_tx, essential_failed_rx) = mpsc::unbounded(); + let (essential_failed_tx, essential_failed_rx) = tracing_unbounded("mpsc_essential_tasks"); let import_queue = Box::new(import_queue); let chain_info = client.chain_info(); - let chain_spec = config.expect_chain_spec(); + let chain_spec = &config.chain_spec; - let version = config.full_version(); - info!("Highest known block at #{}", chain_info.best_number); + let version = config.impl_version; + info!("📦 Highest known block at #{}", chain_info.best_number); telemetry!( SUBSTRATE_INFO; "node.start"; @@ -840,7 +792,7 @@ ServiceBuilder< .register_transaction_pool(Arc::downgrade(&transaction_pool) as _); let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { - imports_external_transactions: !config.roles.is_light(), + imports_external_transactions: !matches!(config.role, Role::Light), pool: transaction_pool.clone(), client: client.clone(), executor: tasks_builder.spawn_handle(), @@ -863,7 +815,7 @@ ServiceBuilder< Box::new(sp_consensus::block_validation::DefaultBlockAnnounceValidator::new(client.clone())); let network_params = sc_network::config::Params { - roles: config.roles, + role: config.role.clone(), executor: { let spawn_handle = tasks_builder.spawn_handle(); Some(Box::new(move |fut| { @@ -913,7 +865,7 @@ ServiceBuilder< let offchain = offchain_workers.as_ref().map(Arc::downgrade); let notifications_spawn_handle = tasks_builder.spawn_handle(); let network_state_info: Arc = network.clone(); - let is_validator = config.roles.is_authority(); + let is_validator = config.role.is_authority(); let (import_stream, finality_stream) = ( client.import_notification_stream().map(|n| ChainEvent::NewBlock { @@ -992,114 +944,44 @@ ServiceBuilder< } // Prometheus metrics. - let metrics = if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() { + let mut metrics_service = if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() { // Set static metrics. - register(Gauge::::with_opts( - Opts::new( - "build_info", - "A metric with a constant '1' value labeled by name, version, and commit." - ) - .const_label("name", config.impl_name) - .const_label("version", config.impl_version) - .const_label("commit", config.impl_commit), - )?, ®istry)?.set(1); - register(Gauge::::new( - "node_roles", "The roles the node is running as", - )?, ®istry)?.set(u64::from(config.roles.bits())); - - let metrics = ServiceMetrics::register(®istry)?; + + let role_bits = match config.role { + Role::Full => 1u64, + Role::Light => 2u64, + Role::Sentry { .. } => 3u64, + Role::Authority { .. } => 4u64, + }; + let metrics = MetricsService::with_prometheus( + ®istry, + &config.network.node_name, + &config.impl_version, + role_bits, + )?; spawn_handle.spawn( "prometheus-endpoint", prometheus_endpoint::init_prometheus(port, registry).map(drop) ); - Some(metrics) + metrics } else { - None + MetricsService::new() }; // Periodically notify the telemetry. let transaction_pool_ = transaction_pool.clone(); let client_ = client.clone(); - let mut sys = System::new(); - let self_pid = get_current_pid().ok(); - let (state_tx, state_rx) = mpsc::unbounded::<(NetworkStatus<_>, NetworkState)>(); + let (state_tx, state_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat1"); network_status_sinks.lock().push(std::time::Duration::from_millis(5000), state_tx); let tel_task = state_rx.for_each(move |(net_status, _)| { let info = client_.usage_info(); - let best_number = info.chain.best_number.saturated_into::(); - let best_hash = info.chain.best_hash; - let num_peers = net_status.num_connected_peers; - let txpool_status = transaction_pool_.status(); - let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); - let bandwidth_download = net_status.average_download_per_sec; - let bandwidth_upload = net_status.average_upload_per_sec; - let best_seen_block = net_status.best_seen_block - .map(|num: NumberFor| num.unique_saturated_into() as u64); - - // get cpu usage and memory usage of this process - let (cpu_usage, memory) = if let Some(self_pid) = self_pid { - if sys.refresh_process(self_pid) { - let proc = sys.get_process(self_pid) - .expect("Above refresh_process succeeds, this should be Some(), qed"); - (proc.cpu_usage(), proc.memory()) - } else { (0.0, 0) } - } else { (0.0, 0) }; - - telemetry!( - SUBSTRATE_INFO; - "system.interval"; - "peers" => num_peers, - "height" => best_number, - "best" => ?best_hash, - "txcount" => txpool_status.ready, - "cpu" => cpu_usage, - "memory" => memory, - "finalized_height" => finalized_number, - "finalized_hash" => ?info.chain.finalized_hash, - "bandwidth_download" => bandwidth_download, - "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => info.usage.as_ref() - .map(|usage| usage.memory.state_cache.as_bytes()) - .unwrap_or(0), - "used_db_cache_size" => info.usage.as_ref() - .map(|usage| usage.memory.database_cache.as_bytes()) - .unwrap_or(0), - "disk_read_per_sec" => info.usage.as_ref() - .map(|usage| usage.io.bytes_read) - .unwrap_or(0), - "disk_write_per_sec" => info.usage.as_ref() - .map(|usage| usage.io.bytes_written) - .unwrap_or(0), + metrics_service.tick( + &info, + &transaction_pool_.status(), + &net_status, ); - if let Some(metrics) = metrics.as_ref() { - metrics.memory_usage_bytes.set(memory); - metrics.cpu_usage_percentage.set(f64::from(cpu_usage)); - metrics.ready_transactions_number.set(txpool_status.ready as u64); - - metrics.network_per_sec_bytes.with_label_values(&["download"]).set(net_status.average_download_per_sec); - metrics.network_per_sec_bytes.with_label_values(&["upload"]).set(net_status.average_upload_per_sec); - - metrics.block_height_number.with_label_values(&["finalized"]).set(finalized_number); - metrics.block_height_number.with_label_values(&["best"]).set(best_number); - - if let Some(best_seen_block) = best_seen_block { - metrics.block_height_number.with_label_values(&["sync_target"]).set(best_seen_block); - } - - if let Some(info) = info.usage.as_ref() { - metrics.database_cache.set(info.memory.database_cache.as_bytes() as u64); - metrics.state_cache.set(info.memory.state_cache.as_bytes() as u64); - - metrics.state_db.with_label_values(&["non_canonical"]).set(info.memory.state_db.non_canonical.as_bytes() as u64); - if let Some(pruning) = info.memory.state_db.pruning { - metrics.state_db.with_label_values(&["pruning"]).set(pruning.as_bytes() as u64); - } - metrics.state_db.with_label_values(&["pinned"]).set(info.memory.state_db.pinned.as_bytes() as u64); - } - } - ready(()) }); @@ -1109,7 +991,7 @@ ServiceBuilder< ); // Periodically send the network state to the telemetry. - let (netstat_tx, netstat_rx) = mpsc::unbounded::<(NetworkStatus<_>, NetworkState)>(); + let (netstat_tx, netstat_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat2"); network_status_sinks.lock().push(std::time::Duration::from_secs(30), netstat_tx); let tel_task_2 = netstat_rx.for_each(move |(_, network_state)| { telemetry!( @@ -1125,7 +1007,7 @@ ServiceBuilder< ); // RPC - let (system_rpc_tx, system_rpc_rx) = mpsc::unbounded(); + let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); let gen_handler = || { use sc_rpc::{chain, state, author, system, offchain}; @@ -1197,25 +1079,26 @@ ServiceBuilder< spawn_handle.spawn( "network-worker", build_network_future( - config.roles, + config.role.clone(), network_mut, client.clone(), network_status_sinks.clone(), system_rpc_rx, has_bootnodes, + config.announce_block, ), ); - let telemetry_connection_sinks: Arc>>> = Default::default(); + let telemetry_connection_sinks: Arc>>> = Default::default(); // Telemetry let telemetry = config.telemetry_endpoints.clone().map(|endpoints| { - let is_authority = config.roles.is_authority(); + let is_authority = config.role.is_authority(); let network_id = network.local_peer_id().to_base58(); - let name = config.name.clone(); + let name = config.network.node_name.clone(); let impl_name = config.impl_name.to_owned(); let version = version.clone(); - let chain_name = config.expect_chain_spec().name().to_owned(); + let chain_name = config.chain_spec.name().to_owned(); let telemetry_connection_sinks_ = telemetry_connection_sinks.clone(); let telemetry = sc_telemetry::init_telemetry(sc_telemetry::TelemetryConfig { endpoints, @@ -1267,7 +1150,7 @@ ServiceBuilder< Ok(Service { client, - task_manager: tasks_builder.into_task_manager(config.task_executor.ok_or(Error::TaskExecutorRequired)?), + task_manager: tasks_builder.into_task_manager(config.task_executor), network, network_status_sinks, select_chain, diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index c58a2755ddc51eeb4e346a5f3c993edbfd5cf578..12fae3224108a66feed5e593e8de50bb97c8a9cf 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -125,7 +125,7 @@ impl< return std::task::Poll::Ready(Err(From::from(err))); }, }; - info!("Importing {} blocks", c); + info!("📦 Importing {} blocks", c); count = Some(c); c } @@ -185,7 +185,7 @@ impl< } if link.imported_blocks >= count { - info!("Imported {} blocks. Best: #{}", read_block_count, client.chain_info().best_number); + info!("🎉 Imported {} blocks. Best: #{}", read_block_count, client.chain_info().best_number); return std::task::Poll::Ready(Ok(())); } else { diff --git a/client/service/src/config.rs b/client/service/src/config.rs index d9d497d1e999f219b95983f65f272137d94d7031..0515a31c7c656aed89386deaa3e9f5335c9cf729 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -18,60 +18,35 @@ pub use sc_client::ExecutionStrategies; pub use sc_client_db::{kvdb::KeyValueDB, PruningMode}; -pub use sc_network::config::{ExtTransport, NetworkConfiguration, Roles}; +pub use sc_network::Multiaddr; +pub use sc_network::config::{ExtTransport, MultiaddrWithPeerId, NetworkConfiguration, Role, NodeKeyConfig}; pub use sc_executor::WasmExecutionMethod; use std::{future::Future, path::{PathBuf, Path}, pin::Pin, net::SocketAddr, sync::Arc}; pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; use sc_chain_spec::ChainSpec; use sp_core::crypto::Protected; -use target_info::Target; -use sc_telemetry::TelemetryEndpoints; +pub use sc_telemetry::TelemetryEndpoints; use prometheus_endpoint::Registry; -/// Executable version. Used to pass version information from the root crate. -#[derive(Clone)] -pub struct VersionInfo { - /// Implementation name. - pub name: &'static str, - /// Implementation version. - pub version: &'static str, - /// SCM Commit hash. - pub commit: &'static str, - /// Executable file name. - pub executable_name: &'static str, - /// Executable file description. - pub description: &'static str, - /// Executable file author. - pub author: &'static str, - /// Support URL. - pub support_url: &'static str, - /// Copyright starting year (x-current year) - pub copyright_start_year: i32, -} - /// Service configuration. pub struct Configuration { /// Implementation name pub impl_name: &'static str, - /// Implementation version + /// Implementation version (see sc-cli to see an example of format) pub impl_version: &'static str, - /// Git commit if any. - pub impl_commit: &'static str, - /// Node roles. - pub roles: Roles, + /// Node role. + pub role: Role, /// How to spawn background tasks. Mandatory, otherwise creating a `Service` will error. - pub task_executor: Option + Send>>) + Send + Sync>>, + pub task_executor: Arc + Send>>) + Send + Sync>, /// Extrinsic pool configuration. pub transaction_pool: TransactionPoolOptions, /// Network configuration. pub network: NetworkConfiguration, - /// Path to the base configuration directory. - pub config_dir: Option, /// Configuration for the keystore. pub keystore: KeystoreConfig, /// Configuration for the database. - pub database: Option, + pub database: DatabaseConfig, /// Size of internal state cache in Bytes pub state_cache_size: usize, /// Size in percent of cache size dedicated to child tries @@ -79,9 +54,7 @@ pub struct Configuration { /// Pruning settings. pub pruning: PruningMode, /// Chain configuration. - pub chain_spec: Option>, - /// Node name. - pub name: String, + pub chain_spec: Box, /// Wasm execution method. pub wasm_method: WasmExecutionMethod, /// Execution strategies. @@ -105,10 +78,6 @@ pub struct Configuration { pub default_heap_pages: Option, /// Should offchain workers be executed. pub offchain_worker: bool, - /// Sentry mode is enabled, the node's role is AUTHORITY but it should not - /// actively participate in consensus (i.e. no keystores should be passed to - /// consensus modules). - pub sentry_mode: bool, /// Enable authoring even when offline. pub force_authoring: bool, /// Disable GRANDPA when running in validator mode @@ -127,13 +96,13 @@ pub struct Configuration { /// /// The default value is 8. pub max_runtime_instances: usize, + /// Announce block automatically after they have been imported + pub announce_block: bool, } /// Configuration of the client keystore. #[derive(Clone)] pub enum KeystoreConfig { - /// No config supplied. - None, /// Keystore at a path on-disk. Recommended for native nodes. Path { /// The path of the keystore. @@ -149,8 +118,8 @@ impl KeystoreConfig { /// Returns the path for the keystore. pub fn path(&self) -> Option<&Path> { match self { - Self::Path { path, .. } => Some(&path), - Self::None | Self::InMemory => None, + Self::Path { path, .. } => Some(path), + Self::InMemory => None, } } } @@ -163,7 +132,7 @@ pub enum DatabaseConfig { /// Path to the database. path: PathBuf, /// Cache Size for internal database in MiB - cache_size: Option, + cache_size: usize, }, /// A custom implementation of an already-open database. @@ -192,129 +161,9 @@ impl PrometheusConfig { } } -impl Default for Configuration { - /// Create a default config - fn default() -> Self { - Configuration { - impl_name: "parity-substrate", - impl_version: "0.0.0", - impl_commit: "", - chain_spec: None, - config_dir: None, - name: Default::default(), - roles: Roles::FULL, - task_executor: None, - transaction_pool: Default::default(), - network: Default::default(), - keystore: KeystoreConfig::None, - database: None, - state_cache_size: Default::default(), - state_cache_child_ratio: Default::default(), - pruning: PruningMode::default(), - wasm_method: WasmExecutionMethod::Interpreted, - execution_strategies: Default::default(), - rpc_http: None, - rpc_ws: None, - rpc_ws_max_connections: None, - rpc_cors: Some(vec![]), - prometheus_config: None, - telemetry_endpoints: None, - telemetry_external_transport: None, - default_heap_pages: None, - offchain_worker: Default::default(), - sentry_mode: false, - force_authoring: false, - disable_grandpa: false, - dev_key_seed: None, - tracing_targets: Default::default(), - tracing_receiver: Default::default(), - max_runtime_instances: 8, - } - } -} - impl Configuration { - /// Create a default config using `VersionInfo` - pub fn from_version(version: &VersionInfo) -> Self { - let mut config = Configuration::default(); - config.impl_name = version.name; - config.impl_version = version.version; - config.impl_commit = version.commit; - - config - } - - /// Returns full version string of this configuration. - pub fn full_version(&self) -> String { - full_version_from_strs(self.impl_version, self.impl_commit) - } - - /// Implementation id and version. - pub fn client_id(&self) -> String { - format!("{}/v{}", self.impl_name, self.full_version()) - } - - /// Generate a PathBuf to sub in the chain configuration directory - /// if given - pub fn in_chain_config_dir(&self, sub: &str) -> Option { - self.config_dir.clone().map(|mut path| { - path.push("chains"); - path.push(self.expect_chain_spec().id()); - path.push(sub); - path - }) - } - - /// Return a reference to the `ChainSpec` of this `Configuration`. - /// - /// ### Panics - /// - /// This method panic if the `chain_spec` is `None` - pub fn expect_chain_spec(&self) -> &dyn ChainSpec { - &**self.chain_spec.as_ref().expect("chain_spec must be specified") - } - - /// Return a reference to the `DatabaseConfig` of this `Configuration`. - /// - /// ### Panics - /// - /// This method panic if the `database` is `None` - pub fn expect_database(&self) -> &DatabaseConfig { - self.database.as_ref().expect("database must be specified") - } - - /// Returns a string displaying the node role, special casing the sentry mode - /// (returning `SENTRY`), since the node technically has an `AUTHORITY` role but - /// doesn't participate. + /// Returns a string displaying the node role. pub fn display_role(&self) -> String { - if self.sentry_mode { - "SENTRY".to_string() - } else { - self.roles.to_string() - } + self.role.to_string() } - - /// Use in memory keystore config when it is not required at all. - /// - /// This function returns an error if the keystore is already set to something different than - /// `KeystoreConfig::None`. - pub fn use_in_memory_keystore(&mut self) -> Result<(), String> { - match &mut self.keystore { - cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, - _ => Err("Keystore config specified when it should not be!".into()), - } - } -} - -/// Returns platform info -pub fn platform() -> String { - let env = Target::env(); - let env_dash = if env.is_empty() { "" } else { "-" }; - format!("{}-{}{}{}", Target::arch(), Target::os(), env_dash, env) -} - -/// Returns full version string, using supplied version and commit. -pub fn full_version_from_strs(impl_version: &str, impl_commit: &str) -> String { - let commit_dash = if impl_commit.is_empty() { "" } else { "-" }; - format!("{}{}{}-{}", impl_version, commit_dash, impl_commit, platform()) } diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 0a7d5ff103f3cf2c28abd7577e4a06ff5e8c797d..16b1d0d718f87128b6bcb8ca884e7527cf8745a2 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -24,6 +24,7 @@ pub mod config; pub mod chain_ops; pub mod error; +mod metrics; mod builder; mod status_sinks; mod task_manager; @@ -40,7 +41,6 @@ use parking_lot::Mutex; use sc_client::Client; use futures::{ Future, FutureExt, Stream, StreamExt, - channel::mpsc, compat::*, sink::SinkExt, task::{Spawn, FutureObj, SpawnError}, @@ -51,6 +51,7 @@ use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT}; use parity_util_mem::MallocSizeOf; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; pub use self::error::Error; pub use self::builder::{ @@ -58,9 +59,10 @@ pub use self::builder::{ ServiceBuilder, ServiceBuilderCommand, TFullClient, TLightClient, TFullBackend, TLightBackend, TFullCallExecutor, TLightCallExecutor, }; -pub use config::{Configuration, Roles, PruningMode}; +pub use config::{Configuration, Role, PruningMode, DatabaseConfig}; pub use sc_chain_spec::{ - ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension + ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, + NoExtension, }; pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoolError}; pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; @@ -71,6 +73,7 @@ pub use sc_executor::NativeExecutionDispatch; pub use std::{ops::Deref, result::Result, sync::Arc}; #[doc(hidden)] pub use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; +pub use sc_tracing::TracingReceiver; pub use task_manager::{TaskManagerBuilder, SpawnTaskHandle}; use task_manager::TaskManager; @@ -98,13 +101,13 @@ pub struct Service { transaction_pool: Arc, /// Send a signal when a spawned essential task has concluded. The next time /// the service future is polled it should complete with an error. - essential_failed_tx: mpsc::UnboundedSender<()>, + essential_failed_tx: TracingUnboundedSender<()>, /// A receiver for spawned essential-tasks concluding. - essential_failed_rx: mpsc::UnboundedReceiver<()>, + essential_failed_rx: TracingUnboundedReceiver<()>, rpc_handlers: sc_rpc_server::RpcHandler, _rpc: Box, _telemetry: Option, - _telemetry_on_connect_sinks: Arc>>>, + _telemetry_on_connect_sinks: Arc>>>, _offchain_workers: Option>, keystore: sc_keystore::KeyStorePtr, marker: PhantomData, @@ -130,7 +133,7 @@ pub trait AbstractService: 'static + Future> + type TransactionPool: TransactionPool + MallocSizeOfWasm; /// Get event stream for telemetry connection established events. - fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()>; + fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()>; /// return a shared instance of Telemetry (if enabled) fn telemetry(&self) -> Option; @@ -171,7 +174,7 @@ pub trait AbstractService: 'static + Future> + -> Arc::Hash>>; /// Returns a receiver that periodically receives a status of the network. - fn network_status(&self, interval: Duration) -> mpsc::UnboundedReceiver<(NetworkStatus, NetworkState)>; + fn network_status(&self, interval: Duration) -> TracingUnboundedReceiver<(NetworkStatus, NetworkState)>; /// Get shared transaction pool instance. fn transaction_pool(&self) -> Arc; @@ -203,8 +206,8 @@ where type SelectChain = TSc; type TransactionPool = TExPool; - fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()> { - let (sink, stream) = futures::channel::mpsc::unbounded(); + fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()> { + let (sink, stream) = tracing_unbounded("mpsc_telemetry_on_connect"); self._telemetry_on_connect_sinks.lock().push(sink); stream } @@ -259,8 +262,8 @@ where self.network.clone() } - fn network_status(&self, interval: Duration) -> mpsc::UnboundedReceiver<(NetworkStatus, NetworkState)> { - let (sink, stream) = mpsc::unbounded(); + fn network_status(&self, interval: Duration) -> TracingUnboundedReceiver<(NetworkStatus, NetworkState)> { + let (sink, stream) = tracing_unbounded("mpsc_network_status"); self.network_status_sinks.lock().push(interval, sink); stream } @@ -322,12 +325,13 @@ fn build_network_future< C: sc_client::BlockchainEvents, H: sc_network::ExHashT > ( - roles: Roles, + role: Role, mut network: sc_network::NetworkWorker, client: Arc, status_sinks: Arc, NetworkState)>>>, - mut rpc_rx: mpsc::UnboundedReceiver>, + mut rpc_rx: TracingUnboundedReceiver>, should_have_peers: bool, + announce_imported_blocks: bool, ) -> impl Future { let mut imported_blocks_stream = client.import_notification_stream().fuse(); let mut finality_notification_stream = client.finality_notification_stream().fuse(); @@ -337,7 +341,11 @@ fn build_network_future< // We poll `imported_blocks_stream`. while let Poll::Ready(Some(notification)) = Pin::new(&mut imported_blocks_stream).poll_next(cx) { - network.on_block_imported(notification.header, Vec::new(), notification.is_new_best); + network.on_block_imported(notification.header, notification.is_new_best); + + if announce_imported_blocks { + network.service().announce_block(notification.hash, Vec::new()); + } } // We poll `finality_notification_stream`, but we only take the last event. @@ -394,17 +402,14 @@ fn build_network_future< sc_rpc::system::Request::NodeRoles(sender) => { use sc_rpc::system::NodeRole; - let node_roles = (0 .. 8) - .filter(|&bit_number| (roles.bits() >> bit_number) & 1 == 1) - .map(|bit_number| match Roles::from_bits(1 << bit_number) { - Some(Roles::AUTHORITY) => NodeRole::Authority, - Some(Roles::LIGHT) => NodeRole::LightClient, - Some(Roles::FULL) => NodeRole::Full, - _ => NodeRole::UnknownRole(bit_number), - }) - .collect(); - - let _ = sender.send(node_roles); + let node_role = match role { + Role::Authority { .. } => NodeRole::Authority, + Role::Light => NodeRole::LightClient, + Role::Full => NodeRole::Full, + Role::Sentry { .. } => NodeRole::Sentry, + }; + + let _ = sender.send(vec![node_role]); } }; } @@ -436,7 +441,7 @@ fn build_network_future< log!( target: "service", if polling_dur >= Duration::from_secs(1) { Level::Warn } else { Level::Trace }, - "Polling the network future took {:?}", + "⚠️ Polling the network future took {:?}", polling_dur ); @@ -621,7 +626,8 @@ where match Decode::decode(&mut &encoded[..]) { Ok(uxt) => { let best_block_id = BlockId::hash(self.client.info().best_hash); - let import_future = self.pool.submit_one(&best_block_id, uxt); + let source = sp_transaction_pool::TransactionSource::External; + let import_future = self.pool.submit_one(&best_block_id, source, uxt); let import_future = import_future .map(move |import_result| { match import_result { @@ -648,7 +654,11 @@ where } fn transaction(&self, hash: &H) -> Option { - self.pool.ready_transaction(hash).map(|tx| tx.data().clone()) + self.pool.ready_transaction(hash) + .and_then( + // Only propagable transactions should be resolved for network service. + |tx| if tx.is_propagable() { Some(tx.data().clone()) } else { None } + ) } } @@ -670,6 +680,7 @@ mod tests { Default::default(), Arc::new(FullChainApi::new(client.clone())), ).0); + let source = sp_runtime::transaction_validity::TransactionSource::External; let best = longest_chain.best_chain().unwrap(); let transaction = Transfer { amount: 5, @@ -677,8 +688,12 @@ mod tests { from: AccountKeyring::Alice.into(), to: Default::default(), }.into_signed_tx(); - block_on(pool.submit_one(&BlockId::hash(best.hash()), transaction.clone())).unwrap(); - block_on(pool.submit_one(&BlockId::hash(best.hash()), Extrinsic::IncludeData(vec![1]))).unwrap(); + block_on(pool.submit_one( + &BlockId::hash(best.hash()), source, transaction.clone()), + ).unwrap(); + block_on(pool.submit_one( + &BlockId::hash(best.hash()), source, Extrinsic::IncludeData(vec![1])), + ).unwrap(); assert_eq!(pool.status().ready, 2); // when diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..6b7c32c2d02612239907b09b9b337d960b36e2f3 --- /dev/null +++ b/client/service/src/metrics.rs @@ -0,0 +1,427 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::convert::TryFrom; + +use crate::NetworkStatus; +use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; +use sc_client::ClientInfo; +use sc_telemetry::{telemetry, SUBSTRATE_INFO}; +use sp_runtime::traits::{NumberFor, Block, SaturatedConversion, UniqueSaturatedInto}; +use sp_transaction_pool::PoolStatus; +use sp_utils::metrics::register_globals; + +use sysinfo::{self, ProcessExt, SystemExt}; + +#[cfg(not(target_os = "unknown"))] +use netstat2::{ + TcpState, ProtocolSocketInfo, iterate_sockets_info, AddressFamilyFlags, ProtocolFlags, +}; + +struct PrometheusMetrics { + // system + #[cfg(any(unix, windows))] + load_avg: GaugeVec, + + // process + cpu_usage_percentage: Gauge, + memory_usage_bytes: Gauge, + threads: Gauge, + open_files: GaugeVec, + + #[cfg(any(unix, windows))] + netstat: GaugeVec, + + // -- inner counters + // generic info + block_height: GaugeVec, + number_leaves: Gauge, + ready_transactions_number: Gauge, + + // I/O + network_per_sec_bytes: GaugeVec, + database_cache: Gauge, + state_cache: Gauge, + state_db: GaugeVec, +} + +impl PrometheusMetrics { + fn setup(registry: &Registry, name: &str, version: &str, roles: u64) + -> Result + { + register(Gauge::::with_opts( + Opts::new( + "build_info", + "A metric with a constant '1' value labeled by name, version" + ) + .const_label("name", name) + .const_label("version", version) + )?, ®istry)?.set(1); + + register(Gauge::::new( + "node_roles", "The roles the node is running as", + )?, ®istry)?.set(roles); + + register_globals(registry)?; + + Ok(Self { + // system + #[cfg(any(unix, windows))] + load_avg: register(GaugeVec::new( + Opts::new("load_avg", "System load average"), + &["over"] + )?, registry)?, + + // process + memory_usage_bytes: register(Gauge::new( + "memory_usage_bytes", "Node memory (resident set size) usage", + )?, registry)?, + + cpu_usage_percentage: register(Gauge::new( + "cpu_usage_percentage", "Node CPU usage", + )?, registry)?, + + #[cfg(any(unix, windows))] + netstat: register(GaugeVec::new( + Opts::new("netstat_tcp", "Current TCP connections "), + &["status"] + )?, registry)?, + + threads: register(Gauge::new( + "threads", "Number of threads used by the process", + )?, registry)?, + + open_files: register(GaugeVec::new( + Opts::new("open_file_handles", "Open file handlers held by the process"), + &["fd_type"] + )?, registry)?, + + // --- internal + + // generic internals + block_height: register(GaugeVec::new( + Opts::new("block_height", "Block height info of the chain"), + &["status"] + )?, registry)?, + + number_leaves: register(Gauge::new( + "number_leaves", "Number of known chain leaves (aka forks)", + )?, registry)?, + + ready_transactions_number: register(Gauge::new( + "ready_transactions_number", "Number of transactions in the ready queue", + )?, registry)?, + + // I/ O + network_per_sec_bytes: register(GaugeVec::new( + Opts::new("network_per_sec_bytes", "Networking bytes per second"), + &["direction"] + )?, registry)?, + database_cache: register(Gauge::new( + "database_cache_bytes", "RocksDB cache size in bytes", + )?, registry)?, + state_cache: register(Gauge::new( + "state_cache_bytes", "State cache size in bytes", + )?, registry)?, + state_db: register(GaugeVec::new( + Opts::new("state_db_cache_bytes", "State DB cache in bytes"), + &["subtype"] + )?, registry)?, + }) + } +} + +#[cfg(any(unix, windows))] +#[derive(Default)] +struct ConnectionsCount { + listen: u64, + established: u64, + starting: u64, + closing: u64, + closed: u64, + other: u64 +} + +#[derive(Default)] +struct FdCounter { + paths: u64, + sockets: u64, + net: u64, + pipes: u64, + anon_inode: u64, + mem: u64, + other: u64, +} + +#[derive(Default)] +struct ProcessInfo { + cpu_usage: f64, + memory: u64, + threads: Option, + open_fd: Option, +} + +pub struct MetricsService { + metrics: Option, + #[cfg(not(target_os = "unknown"))] + system: sysinfo::System, + pid: Option, +} + +#[cfg(target_os = "linux")] +impl MetricsService { + fn inner_new(metrics: Option) -> Self { + let process = procfs::process::Process::myself() + .expect("Procfs doesn't fail on unix. qed"); + + Self { + metrics, + system: sysinfo::System::new(), + pid: Some(process.pid), + } + } + + fn process_info(&mut self) -> ProcessInfo { + let pid = self.pid.clone().expect("unix always has a pid. qed"); + let mut info = self.process_info_for(&pid); + let process = procfs::process::Process::new(pid).expect("Our process exists. qed."); + info.threads = process.stat().ok().map(|s| + u64::try_from(s.num_threads).expect("There are no negative thread counts. qed"), + ); + info.open_fd = process.fd().ok().map(|i| + i.into_iter().fold(FdCounter::default(), |mut f, info| { + match info.target { + procfs::process::FDTarget::Path(_) => f.paths += 1, + procfs::process::FDTarget::Socket(_) => f.sockets += 1, + procfs::process::FDTarget::Net(_) => f.net += 1, + procfs::process::FDTarget::Pipe(_) => f.pipes += 1, + procfs::process::FDTarget::AnonInode(_) => f.anon_inode += 1, + procfs::process::FDTarget::MemFD(_) => f.mem += 1, + procfs::process::FDTarget::Other(_,_) => f.other += 1, + }; + f + }) + ); + info + } +} + +#[cfg(all(any(unix, windows), not(target_os = "linux")))] +impl MetricsService { + fn inner_new(metrics: Option) -> Self { + Self { + metrics, + system: sysinfo::System::new(), + pid: sysinfo::get_current_pid().ok(), + } + } + + fn process_info(&mut self) -> ProcessInfo { + self.pid.map(|pid| self.process_info_for(&pid)).unwrap_or_default() + } +} + + +#[cfg(target_os = "unknown")] +impl MetricsService { + fn inner_new(metrics: Option) -> Self { + Self { + metrics, + pid: None, + } + } + + fn process_info(&mut self) -> ProcessInfo { + ProcessInfo::default() + } +} + + +impl MetricsService { + pub fn with_prometheus(registry: &Registry, name: &str, version: &str, roles: u64) + -> Result + { + PrometheusMetrics::setup(registry, name, version, roles).map(|p| { + Self::inner_new(Some(p)) + }) + } + + pub fn new() -> Self { + Self::inner_new(None) + } + + #[cfg(not(target_os = "unknown"))] + fn process_info_for(&mut self, pid: &sysinfo::Pid) -> ProcessInfo { + let mut info = ProcessInfo::default(); + if self.system.refresh_process(*pid) { + let prc = self.system.get_process(*pid) + .expect("Above refresh_process succeeds, this must be Some(), qed"); + info.cpu_usage = prc.cpu_usage().into(); + info.memory = prc.memory(); + } + info + } + + #[cfg(not(target_os = "unknown"))] + fn connections_info(&self) -> Option { + self.pid.as_ref().and_then(|pid| { + let af_flags = AddressFamilyFlags::IPV4 | AddressFamilyFlags::IPV6; + let proto_flags = ProtocolFlags::TCP; + let netstat_pid = *pid as u32; + + iterate_sockets_info(af_flags, proto_flags).ok().map(|iter| + iter.filter_map(|r| + r.ok().and_then(|s| { + match s.protocol_socket_info { + ProtocolSocketInfo::Tcp(info) + if s.associated_pids.contains(&netstat_pid) => Some(info.state), + _ => None + } + }) + ).fold(ConnectionsCount::default(), |mut counter, socket_state| { + match socket_state { + TcpState::Listen => counter.listen += 1, + TcpState::Established => counter.established += 1, + TcpState::Closed => counter.closed += 1, + TcpState::SynSent | TcpState::SynReceived => counter.starting += 1, + TcpState::FinWait1 | TcpState::FinWait2 | TcpState::CloseWait + | TcpState::Closing | TcpState::LastAck => counter.closing += 1, + _ => counter.other += 1 + } + + counter + }) + ) + }) + } + + pub fn tick( + &mut self, + info: &ClientInfo, + txpool_status: &PoolStatus, + net_status: &NetworkStatus, + ) { + + let best_number = info.chain.best_number.saturated_into::(); + let best_hash = info.chain.best_hash; + let num_peers = net_status.num_connected_peers; + let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); + let bandwidth_download = net_status.average_download_per_sec; + let bandwidth_upload = net_status.average_upload_per_sec; + let best_seen_block = net_status.best_seen_block + .map(|num: NumberFor| num.unique_saturated_into() as u64); + let process_info = self.process_info(); + + telemetry!( + SUBSTRATE_INFO; + "system.interval"; + "peers" => num_peers, + "height" => best_number, + "best" => ?best_hash, + "txcount" => txpool_status.ready, + "cpu" => process_info.cpu_usage, + "memory" => process_info.memory, + "finalized_height" => finalized_number, + "finalized_hash" => ?info.chain.finalized_hash, + "bandwidth_download" => bandwidth_download, + "bandwidth_upload" => bandwidth_upload, + "used_state_cache_size" => info.usage.as_ref() + .map(|usage| usage.memory.state_cache.as_bytes()) + .unwrap_or(0), + "used_db_cache_size" => info.usage.as_ref() + .map(|usage| usage.memory.database_cache.as_bytes()) + .unwrap_or(0), + "disk_read_per_sec" => info.usage.as_ref() + .map(|usage| usage.io.bytes_read) + .unwrap_or(0), + "disk_write_per_sec" => info.usage.as_ref() + .map(|usage| usage.io.bytes_written) + .unwrap_or(0), + ); + + if let Some(metrics) = self.metrics.as_ref() { + metrics.cpu_usage_percentage.set(process_info.cpu_usage as f64); + // `sysinfo::Process::memory` returns memory usage in KiB and not bytes. + metrics.memory_usage_bytes.set(process_info.memory * 1024); + + if let Some(threads) = process_info.threads { + metrics.threads.set(threads); + } + + if let Some(fd_info) = process_info.open_fd { + metrics.open_files.with_label_values(&["paths"]).set(fd_info.paths); + metrics.open_files.with_label_values(&["mem"]).set(fd_info.mem); + metrics.open_files.with_label_values(&["sockets"]).set(fd_info.sockets); + metrics.open_files.with_label_values(&["net"]).set(fd_info.net); + metrics.open_files.with_label_values(&["pipe"]).set(fd_info.pipes); + metrics.open_files.with_label_values(&["anon_inode"]).set(fd_info.anon_inode); + metrics.open_files.with_label_values(&["other"]).set(fd_info.other); + } + + + metrics.network_per_sec_bytes.with_label_values(&["download"]).set( + net_status.average_download_per_sec, + ); + metrics.network_per_sec_bytes.with_label_values(&["upload"]).set( + net_status.average_upload_per_sec, + ); + + metrics.block_height.with_label_values(&["finalized"]).set(finalized_number); + metrics.block_height.with_label_values(&["best"]).set(best_number); + if let Ok(leaves) = u64::try_from(info.chain.number_leaves) { + metrics.number_leaves.set(leaves); + } + + metrics.ready_transactions_number.set(txpool_status.ready as u64); + + if let Some(best_seen_block) = best_seen_block { + metrics.block_height.with_label_values(&["sync_target"]).set(best_seen_block); + } + + if let Some(info) = info.usage.as_ref() { + metrics.database_cache.set(info.memory.database_cache.as_bytes() as u64); + metrics.state_cache.set(info.memory.state_cache.as_bytes() as u64); + + metrics.state_db.with_label_values(&["non_canonical"]).set( + info.memory.state_db.non_canonical.as_bytes() as u64, + ); + if let Some(pruning) = info.memory.state_db.pruning { + metrics.state_db.with_label_values(&["pruning"]).set(pruning.as_bytes() as u64); + } + metrics.state_db.with_label_values(&["pinned"]).set( + info.memory.state_db.pinned.as_bytes() as u64, + ); + } + + #[cfg(not(target_os = "unknown"))] + { + let load = self.system.get_load_average(); + metrics.load_avg.with_label_values(&["1min"]).set(load.one); + metrics.load_avg.with_label_values(&["5min"]).set(load.five); + metrics.load_avg.with_label_values(&["15min"]).set(load.fifteen); + + if let Some(conns) = self.connections_info() { + metrics.netstat.with_label_values(&["listen"]).set(conns.listen); + metrics.netstat.with_label_values(&["established"]).set(conns.established); + metrics.netstat.with_label_values(&["starting"]).set(conns.starting); + metrics.netstat.with_label_values(&["closing"]).set(conns.closing); + metrics.netstat.with_label_values(&["closed"]).set(conns.closed); + metrics.netstat.with_label_values(&["other"]).set(conns.other); + } + } + } + } +} diff --git a/client/service/src/status_sinks.rs b/client/service/src/status_sinks.rs index 8e189be157be5234882d234fc64ed9709a3918a7..4b1dce52f9a319ee02c2033f0214fffd6fe56a86 100644 --- a/client/service/src/status_sinks.rs +++ b/client/service/src/status_sinks.rs @@ -14,11 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use futures::{Stream, stream::futures_unordered::FuturesUnordered, channel::mpsc}; +use futures::{Stream, stream::futures_unordered::FuturesUnordered}; use std::time::Duration; use std::pin::Pin; use std::task::{Poll, Context}; use futures_timer::Delay; +use sp_utils::mpsc::TracingUnboundedSender; /// Holds a list of `UnboundedSender`s, each associated with a certain time period. Every time the /// period elapses, we push an element on the sender. @@ -31,7 +32,7 @@ pub struct StatusSinks { struct YieldAfter { delay: Delay, interval: Duration, - sender: Option>, + sender: Option>, } impl StatusSinks { @@ -45,7 +46,7 @@ impl StatusSinks { /// Adds a sender to the collection. /// /// The `interval` is the time period between two pushes on the sender. - pub fn push(&mut self, interval: Duration, sender: mpsc::UnboundedSender) { + pub fn push(&mut self, interval: Duration, sender: TracingUnboundedSender) { self.entries.push(YieldAfter { delay: Delay::new(interval), interval, @@ -88,7 +89,7 @@ impl StatusSinks { } impl futures::Future for YieldAfter { - type Output = (mpsc::UnboundedSender, Duration); + type Output = (TracingUnboundedSender, Duration); fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { let this = Pin::into_inner(self); diff --git a/client/service/src/task_manager.rs b/client/service/src/task_manager.rs index 15dc33abfad0c315eec4f22fecf36c6c79fb6405..7c5862e8535c4bcacdb846f9f228c630ccfc0c3d 100644 --- a/client/service/src/task_manager.rs +++ b/client/service/src/task_manager.rs @@ -22,17 +22,18 @@ use exit_future::Signal; use log::{debug, error}; use futures::{ Future, FutureExt, Stream, - future::select, channel::mpsc, + future::select, compat::*, task::{Spawn, FutureObj, SpawnError}, }; use sc_client_api::CloneableSpawn; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; /// Type alias for service task executor (usually runtime). pub type ServiceTaskExecutor = Arc + Send>>) + Send + Sync>; /// Type alias for the task scheduler. -pub type TaskScheduler = mpsc::UnboundedSender<(Pin + Send>>, Cow<'static, str>)>; +pub type TaskScheduler = TracingUnboundedSender<(Pin + Send>>, Cow<'static, str>)>; /// Helper struct to setup background tasks execution for service. pub struct TaskManagerBuilder { @@ -44,14 +45,14 @@ pub struct TaskManagerBuilder { /// Sender for futures that must be spawned as background tasks. to_spawn_tx: TaskScheduler, /// Receiver for futures that must be spawned as background tasks. - to_spawn_rx: mpsc::UnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, + to_spawn_rx: TracingUnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, } impl TaskManagerBuilder { /// New asynchronous task manager setup. pub fn new() -> Self { let (signal, on_exit) = exit_future::signal(); - let (to_spawn_tx, to_spawn_rx) = mpsc::unbounded(); + let (to_spawn_tx, to_spawn_rx) = tracing_unbounded("mpsc_task_manager"); Self { on_exit, signal: Some(signal), @@ -144,7 +145,7 @@ pub struct TaskManager { /// Sender for futures that must be spawned as background tasks. to_spawn_tx: TaskScheduler, /// Receiver for futures that must be spawned as background tasks. - to_spawn_rx: mpsc::UnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, + to_spawn_rx: TracingUnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, /// How to spawn background tasks. executor: ServiceTaskExecutor, } diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 401ef8ae04c1da598ffe9f5c019834b3931ca4bd..39c17420bf571fc7688b728999f3e302bf6a7f23 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -23,3 +23,6 @@ sc-client = { version = "0.8.0-alpha.5", path = "../../" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 537d44647701e327037e3a6f5388c15a42b44e0b..bb5ef0956248437211873871f10d582188f766f9 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -34,11 +34,11 @@ use sc_service::{ Configuration, config::{DatabaseConfig, KeystoreConfig}, RuntimeGenesis, - Roles, + Role, Error, }; use sc_network::{multiaddr, Multiaddr, NetworkStateInfo}; -use sc_network::config::{NetworkConfiguration, TransportConfig, NodeKeyConfig, Secret, NonReservedPeerMode}; +use sc_network::config::{NetworkConfiguration, TransportConfig}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_transaction_pool::TransactionPool; @@ -134,7 +134,7 @@ where F: Send + 'static, L: Send +'static, U: Clone + Send + 'static fn node_config ( index: usize, spec: &GenericChainSpec, - role: Roles, + role: Role, task_executor: Arc + Send>>) + Send + Sync>, key_seed: Option, base_port: u16, @@ -143,58 +143,46 @@ fn node_config TestNet where let node_config = node_config( self.nodes, &self.chain_spec, - Roles::AUTHORITY, + Role::Authority { sentry_nodes: Vec::new() }, task_executor, Some(key), self.base_port, @@ -287,7 +275,7 @@ impl TestNet where let executor = executor.clone(); Arc::new(move |fut: Pin + Send>>| executor.spawn(fut.unit_error().compat())) }; - let node_config = node_config(self.nodes, &self.chain_spec, Roles::FULL, task_executor, None, self.base_port, &temp); + let node_config = node_config(self.nodes, &self.chain_spec, Role::Full, task_executor, None, self.base_port, &temp); let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); let (service, user_data) = full(node_config).expect("Error creating test node service"); let service = SyncService::from(service); @@ -303,7 +291,7 @@ impl TestNet where let executor = executor.clone(); Arc::new(move |fut: Pin + Send>>| executor.spawn(fut.unit_error().compat())) }; - let node_config = node_config(self.nodes, &self.chain_spec, Roles::LIGHT, task_executor, None, self.base_port, &temp); + let node_config = node_config(self.nodes, &self.chain_spec, Role::Light, task_executor, None, self.base_port, &temp); let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); let service = SyncService::from(light(node_config).expect("Error creating test node service")); @@ -482,9 +470,10 @@ pub fn sync( let first_user_data = &network.full_nodes[0].2; let best_block = BlockId::number(first_service.get().client().chain_info().best_number); let extrinsic = extrinsic_factory(&first_service.get(), first_user_data); + let source = sp_transaction_pool::TransactionSource::External; futures::executor::block_on( - first_service.get().transaction_pool().submit_one(&best_block, extrinsic) + first_service.get().transaction_pool().submit_one(&best_block, source, extrinsic) ).expect("failed to submit extrinsic"); network.run_until_all_full( diff --git a/client/src/call_executor.rs b/client/src/call_executor.rs index 5a374a81022bdd5ab426bd138da8929a413176e9..1160449eeef88c251673251d873c76855d7bcab3 100644 --- a/client/src/call_executor.rs +++ b/client/src/call_executor.rs @@ -160,36 +160,38 @@ where recorder.clone(), ); - StateMachine::new( + let changes = &mut *changes.borrow_mut(); + let mut state_machine = StateMachine::new( &backend, changes_trie_state, - &mut *changes.borrow_mut(), + changes, &self.executor, method, call_data, extensions.unwrap_or_default(), &runtime_code, self.spawn_handle.clone(), - ) + ); // TODO: https://github.com/paritytech/substrate/issues/4455 // .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) - .execute_using_consensus_failure_handler(execution_manager, native_call) + state_machine.execute_using_consensus_failure_handler(execution_manager, native_call) }, None => { let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); - StateMachine::new( + let runtime_code = state_runtime_code.runtime_code()?; + let changes = &mut *changes.borrow_mut(); + let mut state_machine = StateMachine::new( &state, changes_trie_state, - &mut *changes.borrow_mut(), + changes, &self.executor, method, call_data, extensions.unwrap_or_default(), - &state_runtime_code.runtime_code()?, + &runtime_code, self.spawn_handle.clone(), - ) - .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) - .execute_using_consensus_failure_handler(execution_manager, native_call) + ).with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)); + state_machine.execute_using_consensus_failure_handler(execution_manager, native_call) } }.map_err(Into::into) } diff --git a/client/src/client.rs b/client/src/client.rs index 5c7fa80291ba909f58463baf573f0d1498146e5d..4711ad8b3f1a0e5ac6cdda4628c77abc6e1c9fdc 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -21,7 +21,6 @@ use std::{ result, }; use log::{info, trace, warn}; -use futures::channel::mpsc; use parking_lot::{Mutex, RwLock}; use codec::{Encode, Decode}; use hash_db::Prefix; @@ -78,6 +77,7 @@ pub use sc_client_api::{ notifications::{StorageNotifications, StorageEventStream}, CallExecutor, ExecutorProvider, ProofProvider, CloneableSpawn, }; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_blockchain::Error; use prometheus_endpoint::Registry; @@ -93,8 +93,8 @@ pub struct Client where Block: BlockT { backend: Arc, executor: E, storage_notifications: Mutex>, - import_notification_sinks: Mutex>>>, - finality_notification_sinks: Mutex>>>, + import_notification_sinks: Mutex>>>, + finality_notification_sinks: Mutex>>>, // holds the block hash currently being imported. TODO: replace this with block queue importing_block: RwLock>, block_rules: BlockRules, @@ -260,7 +260,7 @@ impl Client where backend.begin_state_operation(&mut op, BlockId::Hash(Default::default()))?; let state_root = op.reset_storage(genesis_storage)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); - info!("Initializing Genesis block/state (state: {}, header-hash: {})", + info!("🔨 Initializing Genesis block/state (state: {}, header-hash: {})", genesis_block.header().state_root(), genesis_block.header().hash() ); @@ -1764,13 +1764,13 @@ where { /// Get block import event stream. fn import_notification_stream(&self) -> ImportNotifications { - let (sink, stream) = mpsc::unbounded(); + let (sink, stream) = tracing_unbounded("mpsc_import_notification_stream"); self.import_notification_sinks.lock().push(sink); stream } fn finality_notification_stream(&self) -> FinalityNotifications { - let (sink, stream) = mpsc::unbounded(); + let (sink, stream) = tracing_unbounded("mpsc_finality_notification_stream"); self.finality_notification_sinks.lock().push(sink); stream } @@ -3024,7 +3024,7 @@ pub(crate) mod tests { pruning: PruningMode::ArchiveAll, source: DatabaseSettingsSrc::Path { path: tmp.path().into(), - cache_size: None, + cache_size: 128, } }, u64::max_value(), @@ -3226,7 +3226,7 @@ pub(crate) mod tests { pruning: PruningMode::keep_blocks(1), source: DatabaseSettingsSrc::Path { path: tmp.path().into(), - cache_size: None, + cache_size: 128, } }, u64::max_value(), diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index bdbfdbc7ec8f03cba799075dd84274860b018ab3..3672da1822df1b30f377f64a3bf1d8c8d85dd8d7 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -304,6 +304,7 @@ impl HeaderBackend for Blockchain { genesis_hash: storage.genesis_hash, finalized_hash: storage.finalized_hash, finalized_number: storage.finalized_number, + number_leaves: storage.leaves.count() } } diff --git a/client/src/leaves.rs b/client/src/leaves.rs index 1082e6ca071eac8463f0c95163e4e8a86b1a6cb3..7c169488ddcf4ef2031fd4d45a1e121afb3f622e 100644 --- a/client/src/leaves.rs +++ b/client/src/leaves.rs @@ -195,6 +195,11 @@ impl LeafSet where self.storage.iter().flat_map(|(_, hashes)| hashes.iter()).cloned().collect() } + /// Number of known leaves + pub fn count(&self) -> usize { + self.storage.len() + } + /// Write the leaf list to the database transaction. pub fn prepare_transaction(&mut self, tx: &mut DBTransaction, column: u32, prefix: &[u8]) { let mut buf = prefix.to_vec(); diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index 749e24af046f7678da10f270561d5a6c1aa1aeac..0b334d48b7a251a6173f292e9416e0fa7925d6da 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -505,6 +505,12 @@ impl StateBackend for GenesisOrUnavailableState } } + fn register_overlay_stats(&mut self, _stats: &sp_state_machine::StateMachineStats) { } + + fn usage_info(&self) -> sp_state_machine::UsageInfo { + sp_state_machine::UsageInfo::empty() + } + fn as_trie_backend(&mut self) -> Option<&TrieBackend> { match self { GenesisOrUnavailableState::Genesis(ref mut state) => state.as_trie_backend(), diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index c056ee9fac8d6c061b569ab0a0c3bd8f1c958cb0..9da3c4e1274cd6065999b2a0214280b4b44ce87d 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -13,9 +13,12 @@ parking_lot = "0.10.0" log = "0.4.8" sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parity-util-mem = "0.6" parity-util-mem-derive = "0.1.0" [dev-dependencies] env_logger = "0.7.0" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 543abd6a8b091a268b0856a00681b7d957d35e17..dcf218e9763fdb9cb39e7a7877881c7915a5d055 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -13,10 +13,10 @@ documentation = "https://docs.rs/sc-telemetry" [dependencies] bytes = "0.5" parking_lot = "0.10.0" -futures = "0.3.1" +futures = "0.3.4" futures-timer = "3.0.1" wasm-timer = "0.2.0" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.17.0", default-features = false, features = ["websocket", "wasm-ext", "tcp", "dns"] } log = "0.4.8" pin-project = "0.4.6" rand = "0.7.2" @@ -26,3 +26,6 @@ slog-json = { version = "2.3.0", features = ["nested-values"] } slog-scope = "4.1.2" take_mut = "0.2.2" void = "1.0.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/telemetry/src/lib.rs b/client/telemetry/src/lib.rs index f8ca6d5c73d41bdd86dff66a2ebbff8dc7d33dbc..6c90d6bbcca9f3a60fc765e056abd908c0ea0636 100644 --- a/client/telemetry/src/lib.rs +++ b/client/telemetry/src/lib.rs @@ -22,7 +22,7 @@ //! `slog_scope::with_logger` followed with `slog_log!`. //! //! Note that you are supposed to only ever use `telemetry!` and not `slog_scope::with_logger` at -//! the moment. Substate may eventually be reworked to get proper `slog` support, including sending +//! the moment. Substrate may eventually be reworked to get proper `slog` support, including sending //! information to the telemetry. //! //! The [`Telemetry`] struct implements `Stream` and must be polled regularly (or sent to a @@ -41,7 +41,7 @@ //! endpoints: sc_telemetry::TelemetryEndpoints::new(vec![ //! // The `0` is the maximum verbosity level of messages to send to this endpoint. //! ("wss://example.com".into(), 0) -//! ]), +//! ]).expect("Invalid URL or multiaddr provided"), //! // Can be used to pass an external implementation of WebSockets. //! wasm_external_transport: None, //! }); @@ -62,7 +62,7 @@ use futures::{prelude::*, channel::mpsc}; use libp2p::{Multiaddr, wasm_ext}; use log::{error, warn}; use parking_lot::Mutex; -use serde::{Serialize, Deserialize}; +use serde::{Serialize, Deserialize, Deserializer}; use std::{pin::Pin, sync::Arc, task::{Context, Poll}, time::Duration}; use wasm_timer::Instant; @@ -96,12 +96,47 @@ pub struct TelemetryConfig { /// /// The URL string can be either a URL or a multiaddress. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct TelemetryEndpoints(Vec<(String, u8)>); +pub struct TelemetryEndpoints( + #[serde(deserialize_with = "url_or_multiaddr_deser")] + Vec<(Multiaddr, u8)> +); + +/// Custom deserializer for TelemetryEndpoints, used to convert urls or multiaddr to multiaddr. +fn url_or_multiaddr_deser<'de, D>(deserializer: D) -> Result, D::Error> + where D: Deserializer<'de> +{ + Vec::<(String, u8)>::deserialize(deserializer)? + .iter() + .map(|e| Ok((url_to_multiaddr(&e.0) + .map_err(serde::de::Error::custom)?, e.1))) + .collect() +} impl TelemetryEndpoints { - pub fn new(endpoints: Vec<(String, u8)>) -> Self { - TelemetryEndpoints(endpoints) + pub fn new(endpoints: Vec<(String, u8)>) -> Result { + let endpoints: Result, libp2p::multiaddr::Error> = endpoints.iter() + .map(|e| Ok((url_to_multiaddr(&e.0)?, e.1))) + .collect(); + endpoints.map(Self) + } +} + +/// Parses a WebSocket URL into a libp2p `Multiaddr`. +fn url_to_multiaddr(url: &str) -> Result { + // First, assume that we have a `Multiaddr`. + let parse_error = match url.parse() { + Ok(ma) => return Ok(ma), + Err(err) => err, + }; + + // If not, try the `ws://path/url` format. + if let Ok(ma) = libp2p::multiaddr::from_url(url) { + return Ok(ma) } + + // If we have no clue about the format of that string, assume that we were expecting a + // `Multiaddr`. + Err(parse_error) } /// Log levels. @@ -149,13 +184,7 @@ struct TelemetryDrain { /// doesn't provide any way of knowing whether a global logger has already been registered. pub fn init_telemetry(config: TelemetryConfig) -> Telemetry { // Build the list of telemetry endpoints. - let mut endpoints = Vec::new(); - for &(ref url, verbosity) in &config.endpoints.0 { - match url_to_multiaddr(url) { - Ok(addr) => endpoints.push((addr, verbosity)), - Err(err) => warn!(target: "telemetry", "Invalid telemetry URL {}: {}", url, err), - } - } + let (endpoints, wasm_external_transport) = (config.endpoints.0, config.wasm_external_transport); let (sender, receiver) = mpsc::channel(16); let guard = { @@ -164,7 +193,7 @@ pub fn init_telemetry(config: TelemetryConfig) -> Telemetry { slog_scope::set_global_logger(root) }; - let worker = match worker::TelemetryWorker::new(endpoints, config.wasm_external_transport) { + let worker = match worker::TelemetryWorker::new(endpoints, wasm_external_transport) { Ok(w) => Some(w), Err(err) => { error!(target: "telemetry", "Failed to initialize telemetry worker: {:?}", err); @@ -271,24 +300,6 @@ impl slog::Drain for TelemetryDrain { } } -/// Parses a WebSocket URL into a libp2p `Multiaddr`. -fn url_to_multiaddr(url: &str) -> Result { - // First, assume that we have a `Multiaddr`. - let parse_error = match url.parse() { - Ok(ma) => return Ok(ma), - Err(err) => err, - }; - - // If not, try the `ws://path/url` format. - if let Ok(ma) = libp2p::multiaddr::from_url(url) { - return Ok(ma) - } - - // If we have no clue about the format of that string, assume that we were expecting a - // `Multiaddr`. - Err(parse_error) -} - /// Translates to `slog_scope::info`, but contains an additional verbosity /// parameter which the log record is tagged with. Additionally the verbosity /// parameter is added to the record as a key-value pair. @@ -300,3 +311,35 @@ macro_rules! telemetry { }) } } + +#[cfg(test)] +mod telemetry_endpoints_tests { + use libp2p::Multiaddr; + use super::TelemetryEndpoints; + use super::url_to_multiaddr; + + #[test] + fn valid_endpoints() { + let endp = vec![("wss://telemetry.polkadot.io/submit/".into(), 3), ("/ip4/80.123.90.4/tcp/5432".into(), 4)]; + let telem = TelemetryEndpoints::new(endp.clone()).expect("Telemetry endpoint should be valid"); + let mut res: Vec<(Multiaddr, u8)> = vec![]; + for (a, b) in endp.iter() { + res.push((url_to_multiaddr(a).expect("provided url should be valid"), *b)) + } + assert_eq!(telem.0, res); + } + + #[test] + fn invalid_endpoints() { + let endp = vec![("/ip4/...80.123.90.4/tcp/5432".into(), 3), ("/ip4/no:!?;rlkqre;;::::///tcp/5432".into(), 4)]; + let telem = TelemetryEndpoints::new(endp); + assert!(telem.is_err()); + } + + #[test] + fn valid_and_invalid_endpoints() { + let endp = vec![("/ip4/80.123.90.4/tcp/5432".into(), 3), ("/ip4/no:!?;rlkqre;;::::///tcp/5432".into(), 4)]; + let telem = TelemetryEndpoints::new(endp); + assert!(telem.is_err()); + } +} diff --git a/client/telemetry/src/worker/node.rs b/client/telemetry/src/worker/node.rs index 58e9f20bd5f7c7eb221ad2373be5a414ba4154a4..454f504d660050995cba1f6c122c311ba35741ee 100644 --- a/client/telemetry/src/worker/node.rs +++ b/client/telemetry/src/worker/node.rs @@ -116,7 +116,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, pending.push_back(payload.into()); Ok(()) } else { - warn!(target: "telemetry", "Rejected log entry because queue is full for {:?}", + warn!(target: "telemetry", "⚠️ Rejected log entry because queue is full for {:?}", self.addr); Err(()) } @@ -137,7 +137,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, break NodeSocket::Connected(conn) }, Poll::Ready(Err(err)) => { - warn!(target: "telemetry", "Disconnected from {}: {:?}", self.addr, err); + warn!(target: "telemetry", "⚠️ Disconnected from {}: {:?}", self.addr, err); let timeout = gen_rand_reconnect_delay(); self.socket = NodeSocket::WaitingReconnect(timeout); return Poll::Ready(NodeEvent::Disconnected(err)) @@ -146,7 +146,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, } NodeSocket::Dialing(mut s) => match Future::poll(Pin::new(&mut s), cx) { Poll::Ready(Ok(sink)) => { - debug!(target: "telemetry", "Connected to {}", self.addr); + debug!(target: "telemetry", "✅ Connected to {}", self.addr); let conn = NodeSocketConnected { sink, pending: VecDeque::new(), @@ -158,7 +158,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, }, Poll::Pending => break NodeSocket::Dialing(s), Poll::Ready(Err(err)) => { - warn!(target: "telemetry", "Error while dialing {}: {:?}", self.addr, err); + warn!(target: "telemetry", "❌ Error while dialing {}: {:?}", self.addr, err); let timeout = gen_rand_reconnect_delay(); socket = NodeSocket::WaitingReconnect(timeout); } @@ -169,7 +169,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, socket = NodeSocket::Dialing(d); } Err(err) => { - warn!(target: "telemetry", "Error while dialing {}: {:?}", self.addr, err); + warn!(target: "telemetry", "❌ Error while dialing {}: {:?}", self.addr, err); let timeout = gen_rand_reconnect_delay(); socket = NodeSocket::WaitingReconnect(timeout); } @@ -181,7 +181,7 @@ where TTrans: Clone + Unpin, TTrans::Dial: Unpin, break NodeSocket::WaitingReconnect(s) } NodeSocket::Poisoned => { - error!(target: "telemetry", "Poisoned connection with {}", self.addr); + error!(target: "telemetry", "‼️ Poisoned connection with {}", self.addr); break NodeSocket::Poisoned } } diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index 60a449c687cc236054cbfb99e88f3c5fd4a2d105..319526b61044858240908719d233aaef5a134f98 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -21,3 +21,6 @@ sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } [dev-dependencies] tracing = "0.1.10" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index ca7130b2d65b091897280d44d8c8a01532c9516a..29b80698424e1a60a8d4fc542d6dea3354706910 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate transaction pool implementation." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" @@ -19,14 +19,20 @@ wasm-timer = "0.2" sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } sc-transaction-graph = { version = "2.0.0-alpha.5", path = "./graph" } sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -futures-timer = "2.0" +intervalier = "0.3.1" parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } [dev-dependencies] +assert_matches = "1.3.0" +hex = "0.4" sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } substrate-test-runtime-transaction-pool = { version = "2.0.0-dev", path = "../../test-utils/runtime/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index 08593e67845cca8a85b1e7ce9f416285d90a310f..df2fd8546a2650453d273d558bf6b17355509b5c 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -10,12 +10,13 @@ description = "Generic Transaction Pool" [dependencies] derive_more = "0.99.2" -futures = "0.3.1" +futures = "0.3.4" log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } wasm-timer = "0.2" sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } +sp-utils = { version = "2.0.0-alpha.5", path = "../../../primitives/utils" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } @@ -24,10 +25,13 @@ linked-hash-map = "0.5.2" [dev-dependencies] assert_matches = "1.3.0" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } criterion = "0.3" [[bench]] name = "basics" harness = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/transaction-pool/graph/benches/basics.rs b/client/transaction-pool/graph/benches/basics.rs index 6f5d39f09f5c540a31762cd129e89f48bb4c66d1..23b4dba3488061429a39c04bcd717b55ee007c78 100644 --- a/client/transaction-pool/graph/benches/basics.rs +++ b/client/transaction-pool/graph/benches/basics.rs @@ -18,12 +18,14 @@ use criterion::{criterion_group, criterion_main, Criterion}; use futures::{future::{ready, Ready}, executor::block_on}; use sc_transaction_graph::*; -use sp_runtime::transaction_validity::{ValidTransaction, InvalidTransaction}; use codec::Encode; use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId}; use sp_runtime::{ generic::BlockId, - transaction_validity::{TransactionValidity, TransactionTag as Tag}, + transaction_validity::{ + ValidTransaction, InvalidTransaction, TransactionValidity, TransactionTag as Tag, + TransactionSource, + }, }; use sp_core::blake2_256; @@ -55,6 +57,7 @@ impl ChainApi for TestApi { fn validate_transaction( &self, at: &BlockId, + _source: TransactionSource, uxt: ExtrinsicFor, ) -> Self::ValidationFuture { let nonce = uxt.transfer().nonce; @@ -121,6 +124,7 @@ fn uxt(transfer: Transfer) -> Extrinsic { } fn bench_configured(pool: Pool, number: u64) { + let source = TransactionSource::External; let mut futures = Vec::new(); let mut tags = Vec::new(); @@ -133,7 +137,7 @@ fn bench_configured(pool: Pool, number: u64) { }); tags.push(to_tag(nonce, AccountId::from_h256(H256::from_low_u64_be(1)))); - futures.push(pool.submit_one(&BlockId::Number(1), xt)); + futures.push(pool.submit_one(&BlockId::Number(1), source, xt)); } let res = block_on(futures::future::join_all(futures.into_iter())); diff --git a/client/transaction-pool/graph/src/base_pool.rs b/client/transaction-pool/graph/src/base_pool.rs index 33b7a51f9415b5bc04a288e3a773e817178b4130..38151e9bfd23b87b3fd4511022eaa7f9953ab46a 100644 --- a/client/transaction-pool/graph/src/base_pool.rs +++ b/client/transaction-pool/graph/src/base_pool.rs @@ -33,6 +33,7 @@ use sp_runtime::transaction_validity::{ TransactionTag as Tag, TransactionLongevity as Longevity, TransactionPriority as Priority, + TransactionSource as Source, }; use sp_transaction_pool::{error, PoolStatus, InPoolTransaction}; @@ -102,6 +103,8 @@ pub struct Transaction { pub provides: Vec, /// Should that transaction be propagated. pub propagate: bool, + /// Source of that transaction. + pub source: Source, } impl AsRef for Transaction { @@ -155,6 +158,7 @@ impl Transaction { bytes: self.bytes.clone(), hash: self.hash.clone(), priority: self.priority.clone(), + source: self.source, valid_till: self.valid_till.clone(), requires: self.requires.clone(), provides: self.provides.clone(), @@ -185,6 +189,7 @@ impl fmt::Debug for Transaction where write!(fmt, "valid_till: {:?}, ", &self.valid_till)?; write!(fmt, "bytes: {:?}, ", &self.bytes)?; write!(fmt, "propagate: {:?}, ", &self.propagate)?; + write!(fmt, "source: {:?}, ", &self.source)?; write!(fmt, "requires: [")?; print_tags(fmt, &self.requires)?; write!(fmt, "], provides: [")?; @@ -556,6 +561,7 @@ mod tests { requires: vec![], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); // then @@ -578,6 +584,7 @@ mod tests { requires: vec![], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![1u8], @@ -588,6 +595,7 @@ mod tests { requires: vec![], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap_err(); // then @@ -611,6 +619,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 0); assert_eq!(pool.ready.len(), 0); @@ -623,6 +632,7 @@ mod tests { requires: vec![], provides: vec![vec![0]], propagate: true, + source: Source::External, }).unwrap(); // then @@ -645,6 +655,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![3u8], @@ -655,6 +666,7 @@ mod tests { requires: vec![vec![2]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![2u8], @@ -665,6 +677,7 @@ mod tests { requires: vec![vec![1]], provides: vec![vec![3], vec![2]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![4u8], @@ -675,6 +688,7 @@ mod tests { requires: vec![vec![3], vec![4]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 0); assert_eq!(pool.ready.len(), 0); @@ -688,6 +702,7 @@ mod tests { requires: vec![], provides: vec![vec![0], vec![4]], propagate: true, + source: Source::External, }).unwrap(); // then @@ -720,6 +735,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![3u8], @@ -730,6 +746,7 @@ mod tests { requires: vec![vec![1]], provides: vec![vec![2]], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 0); assert_eq!(pool.ready.len(), 0); @@ -744,6 +761,7 @@ mod tests { requires: vec![vec![2]], provides: vec![vec![0]], propagate: true, + source: Source::External, }).unwrap(); // then @@ -764,6 +782,7 @@ mod tests { requires: vec![], provides: vec![vec![0]], propagate: true, + source: Source::External, }).unwrap(); let mut it = pool.ready().into_iter().map(|tx| tx.data[0]); assert_eq!(it.next(), Some(4)); @@ -792,6 +811,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![3u8], @@ -802,6 +822,7 @@ mod tests { requires: vec![vec![1]], provides: vec![vec![2]], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 0); assert_eq!(pool.ready.len(), 0); @@ -816,6 +837,7 @@ mod tests { requires: vec![vec![2]], provides: vec![vec![0]], propagate: true, + source: Source::External, }).unwrap(); // then @@ -836,6 +858,7 @@ mod tests { requires: vec![], provides: vec![vec![0]], propagate: true, + source: Source::External, }).unwrap_err(); let mut it = pool.ready().into_iter().map(|tx| tx.data[0]); assert_eq!(it.next(), None); @@ -859,6 +882,7 @@ mod tests { requires: vec![], provides: vec![vec![0], vec![4]], propagate: true, + source: Source::External, }).expect("import 1 should be ok"); pool.import(Transaction { data: vec![3u8; 1024], @@ -869,6 +893,7 @@ mod tests { requires: vec![], provides: vec![vec![2], vec![7]], propagate: true, + source: Source::External, }).expect("import 2 should be ok"); assert!(parity_util_mem::malloc_size(&pool) > 5000); @@ -887,6 +912,7 @@ mod tests { requires: vec![], provides: vec![vec![0], vec![4]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![1u8], @@ -897,6 +923,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![3u8], @@ -907,6 +934,7 @@ mod tests { requires: vec![vec![2]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![2u8], @@ -917,6 +945,7 @@ mod tests { requires: vec![vec![1]], provides: vec![vec![3], vec![2]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![4u8], @@ -927,6 +956,7 @@ mod tests { requires: vec![vec![3], vec![4]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); // future pool.import(Transaction { @@ -938,6 +968,7 @@ mod tests { requires: vec![vec![11]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 5); assert_eq!(pool.future.len(), 1); @@ -964,6 +995,7 @@ mod tests { requires: vec![vec![0]], provides: vec![vec![100]], propagate: true, + source: Source::External, }).unwrap(); // ready pool.import(Transaction { @@ -975,6 +1007,7 @@ mod tests { requires: vec![], provides: vec![vec![1]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![2u8], @@ -985,6 +1018,7 @@ mod tests { requires: vec![vec![2]], provides: vec![vec![3]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![3u8], @@ -995,6 +1029,7 @@ mod tests { requires: vec![vec![1]], provides: vec![vec![2]], propagate: true, + source: Source::External, }).unwrap(); pool.import(Transaction { data: vec![4u8], @@ -1005,6 +1040,7 @@ mod tests { requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: true, + source: Source::External, }).unwrap(); assert_eq!(pool.ready().count(), 4); @@ -1040,10 +1076,11 @@ mod tests { requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: true, + source: Source::External, }), "Transaction { \ hash: 4, priority: 1000, valid_till: 64, bytes: 1, propagate: true, \ -requires: [03,02], provides: [04], data: [4]}".to_owned() +source: TransactionSource::External, requires: [03,02], provides: [04], data: [4]}".to_owned() ); } @@ -1058,6 +1095,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: true, + source: Source::External, }.is_propagable(), true); assert_eq!(Transaction { @@ -1069,6 +1107,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: false, + source: Source::External, }.is_propagable(), false); } @@ -1090,6 +1129,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![0]], provides: vec![], propagate: true, + source: Source::External, }); if let Err(error::Error::RejectedFutureTransaction) = err { @@ -1113,6 +1153,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![0]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); // then @@ -1142,6 +1183,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![0]], provides: vec![], propagate: true, + source: Source::External, }).unwrap(); flag diff --git a/client/transaction-pool/graph/src/future.rs b/client/transaction-pool/graph/src/future.rs index a84a5fbe689c37aacbba24ed1e6a8c6885961589..76181c837f988586d72ebd3568b4c31a35ac6b8d 100644 --- a/client/transaction-pool/graph/src/future.rs +++ b/client/transaction-pool/graph/src/future.rs @@ -249,6 +249,7 @@ impl FutureTransactions { #[cfg(test)] mod tests { use super::*; + use sp_runtime::transaction_validity::TransactionSource; #[test] fn can_track_heap_size() { @@ -263,6 +264,7 @@ mod tests { requires: vec![vec![1], vec![2]], provides: vec![vec![3], vec![4]], propagate: true, + source: TransactionSource::External, }.into(), missing_tags: vec![vec![1u8], vec![2u8]].into_iter().collect(), imported_at: std::time::Instant::now(), diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index f0bf17dcb8dd2ecb2091dc9a799c06e8ac90f879..0b817b155d8ecaef12aee3461e0e475d20b15a86 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -24,23 +24,23 @@ use crate::base_pool as base; use crate::watcher::Watcher; use serde::Serialize; -use futures::{ - Future, FutureExt, - channel::mpsc, -}; +use futures::{Future, FutureExt}; use sp_runtime::{ generic::BlockId, traits::{self, SaturatedConversion}, - transaction_validity::{TransactionValidity, TransactionTag as Tag, TransactionValidityError}, + transaction_validity::{ + TransactionValidity, TransactionTag as Tag, TransactionValidityError, TransactionSource, + }, }; use sp_transaction_pool::error; use wasm_timer::Instant; +use sp_utils::mpsc::TracingUnboundedReceiver; use crate::validated_pool::ValidatedPool; pub use crate::validated_pool::ValidatedTransaction; /// Modification notification event stream type; -pub type EventStream = mpsc::UnboundedReceiver; +pub type EventStream = TracingUnboundedReceiver; /// Extrinsic hash type for a pool. pub type ExHash = ::Hash; @@ -76,6 +76,7 @@ pub trait ChainApi: Send + Sync { fn validate_transaction( &self, at: &BlockId, + source: TransactionSource, uxt: ExtrinsicFor, ) -> Self::ValidationFuture; @@ -144,12 +145,17 @@ impl Pool { } /// Imports a bunch of unverified extrinsics to the pool - pub async fn submit_at(&self, at: &BlockId, xts: T, force: bool) - -> Result, B::Error>>, B::Error> - where - T: IntoIterator> + pub async fn submit_at( + &self, + at: &BlockId, + source: TransactionSource, + xts: T, + force: bool, + ) -> Result, B::Error>>, B::Error> where + T: IntoIterator>, { let validated_pool = self.validated_pool.clone(); + let xts = xts.into_iter().map(|xt| (source, xt)); self.verify(at, xts, force) .map(move |validated_transactions| validated_transactions .map(|validated_transactions| validated_pool.submit(validated_transactions @@ -162,9 +168,10 @@ impl Pool { pub async fn submit_one( &self, at: &BlockId, + source: TransactionSource, xt: ExtrinsicFor, ) -> Result, B::Error> { - self.submit_at(at, std::iter::once(xt), false) + self.submit_at(at, source, std::iter::once(xt), false) .map(|import_result| import_result.and_then(|mut import_result| import_result .pop() .expect("One extrinsic passed; one result returned; qed") @@ -176,10 +183,13 @@ impl Pool { pub async fn submit_and_watch( &self, at: &BlockId, + source: TransactionSource, xt: ExtrinsicFor, ) -> Result, BlockHash>, B::Error> { let block_number = self.resolve_block_number(at)?; - let (_, tx) = self.verify_one(at, block_number, xt, false).await; + let (_, tx) = self.verify_one( + at, block_number, source, xt, false + ).await; self.validated_pool.submit_and_watch(tx) } @@ -249,7 +259,7 @@ impl Pool { // to get validity info and tags that the extrinsic provides. None => { let validity = self.validated_pool.api() - .validate_transaction(parent, extrinsic.clone()) + .validate_transaction(parent, TransactionSource::InBlock, extrinsic.clone()) .await; if let Ok(Ok(validity)) = validity { @@ -303,8 +313,12 @@ impl Pool { // Try to re-validate pruned transactions since some of them might be still valid. // note that `known_imported_hashes` will be rejected here due to temporary ban. - let pruned_hashes = prune_status.pruned.iter().map(|tx| tx.hash.clone()).collect::>(); - let pruned_transactions = prune_status.pruned.into_iter().map(|tx| tx.data.clone()); + let pruned_hashes = prune_status.pruned + .iter() + .map(|tx| tx.hash.clone()).collect::>(); + let pruned_transactions = prune_status.pruned + .into_iter() + .map(|tx| (tx.source, tx.data.clone())); let reverified_transactions = self.verify(at, pruned_transactions, false).await?; @@ -335,7 +349,7 @@ impl Pool { async fn verify( &self, at: &BlockId, - xts: impl IntoIterator>, + xts: impl IntoIterator)>, force: bool, ) -> Result, ValidatedTransactionFor>, B::Error> { // we need a block number to compute tx validity @@ -345,7 +359,7 @@ impl Pool { for (hash, validated_tx) in futures::future::join_all( xts.into_iter() - .map(|xt| self.verify_one(at, block_number, xt, force)) + .map(|(source, xt)| self.verify_one(at, block_number, source, xt, force)) ) .await { @@ -360,6 +374,7 @@ impl Pool { &self, block_id: &BlockId, block_number: NumberFor, + source: TransactionSource, xt: ExtrinsicFor, force: bool, ) -> (ExHash, ValidatedTransactionFor) { @@ -371,7 +386,11 @@ impl Pool { ) } - let validation_result = self.validated_pool.api().validate_transaction(block_id, xt.clone()).await; + let validation_result = self.validated_pool.api().validate_transaction( + block_id, + source, + xt.clone(), + ).await; let status = match validation_result { Ok(status) => status, @@ -386,6 +405,7 @@ impl Pool { ValidatedTransaction::valid_at( block_number.saturated_into::(), hash.clone(), + source, xt, bytes, validity, @@ -422,7 +442,7 @@ mod tests { use futures::executor::block_on; use super::*; use sp_transaction_pool::TransactionStatus; - use sp_runtime::transaction_validity::{ValidTransaction, InvalidTransaction}; + use sp_runtime::transaction_validity::{ValidTransaction, InvalidTransaction, TransactionSource}; use codec::Encode; use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId}; use assert_matches::assert_matches; @@ -430,6 +450,7 @@ mod tests { use crate::base_pool::Limit; const INVALID_NONCE: u64 = 254; + const SOURCE: TransactionSource = TransactionSource::External; #[derive(Clone, Debug, Default)] struct TestApi { @@ -450,6 +471,7 @@ mod tests { fn validate_transaction( &self, at: &BlockId, + _source: TransactionSource, uxt: ExtrinsicFor, ) -> Self::ValidationFuture { let hash = self.hash_and_length(&uxt).0; @@ -541,7 +563,7 @@ mod tests { let pool = pool(); // when - let hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -565,7 +587,7 @@ mod tests { // when pool.validated_pool.rotator().ban(&Instant::now(), vec![pool.hash_of(&uxt)]); - let res = block_on(pool.submit_one(&BlockId::Number(0), uxt)); + let res = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt)); assert_eq!(pool.validated_pool().status().ready, 0); assert_eq!(pool.validated_pool().status().future, 0); @@ -581,20 +603,20 @@ mod tests { let stream = pool.validated_pool().import_notification_stream(); // when - let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let _hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, nonce: 0, }))).unwrap(); - let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let _hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, nonce: 1, }))).unwrap(); // future doesn't count - let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let _hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -617,19 +639,19 @@ mod tests { fn should_clear_stale_transactions() { // given let pool = pool(); - let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash1 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, nonce: 0, }))).unwrap(); - let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash2 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, nonce: 1, }))).unwrap(); - let hash3 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash3 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -653,7 +675,7 @@ mod tests { fn should_ban_mined_transactions() { // given let pool = pool(); - let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash1 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -680,7 +702,7 @@ mod tests { ..Default::default() }, TestApi::default().into()); - let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash1 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -689,7 +711,7 @@ mod tests { assert_eq!(pool.validated_pool().status().future, 1); // when - let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let hash2 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(2)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -716,7 +738,7 @@ mod tests { }, TestApi::default().into()); // when - block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -734,7 +756,7 @@ mod tests { let pool = pool(); // when - let err = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + let err = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -754,7 +776,7 @@ mod tests { fn should_trigger_ready_and_finalized() { // given let pool = pool(); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer { + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -778,7 +800,7 @@ mod tests { fn should_trigger_ready_and_finalized_when_pruning_via_hash() { // given let pool = pool(); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer { + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -802,7 +824,7 @@ mod tests { fn should_trigger_future_and_ready_after_promoted() { // given let pool = pool(); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer { + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -812,7 +834,7 @@ mod tests { assert_eq!(pool.validated_pool().status().future, 1); // when - block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { + block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -836,7 +858,7 @@ mod tests { amount: 5, nonce: 0, }); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, uxt)).unwrap(); assert_eq!(pool.validated_pool().status().ready, 1); // when @@ -860,7 +882,7 @@ mod tests { amount: 5, nonce: 0, }); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, uxt)).unwrap(); assert_eq!(pool.validated_pool().status().ready, 1); // when @@ -895,7 +917,7 @@ mod tests { amount: 5, nonce: 0, }); - let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), xt)).unwrap(); + let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, xt)).unwrap(); assert_eq!(pool.validated_pool().status().ready, 1); // when @@ -905,7 +927,7 @@ mod tests { amount: 4, nonce: 1, }); - block_on(pool.submit_one(&BlockId::Number(1), xt)).unwrap(); + block_on(pool.submit_one(&BlockId::Number(1), SOURCE, xt)).unwrap(); assert_eq!(pool.validated_pool().status().ready, 1); // then @@ -934,7 +956,7 @@ mod tests { // This transaction should go to future, since we use `nonce: 1` let pool2 = pool.clone(); std::thread::spawn(move || { - block_on(pool2.submit_one(&BlockId::Number(0), xt)).unwrap(); + block_on(pool2.submit_one(&BlockId::Number(0), SOURCE, xt)).unwrap(); ready.send(()).unwrap(); }); @@ -948,7 +970,7 @@ mod tests { }); // The tag the above transaction provides (TestApi is using just nonce as u8) let provides = vec![0_u8]; - block_on(pool.submit_one(&BlockId::Number(0), xt)).unwrap(); + block_on(pool.submit_one(&BlockId::Number(0), SOURCE, xt)).unwrap(); assert_eq!(pool.validated_pool().status().ready, 1); // Now block import happens before the second transaction is able to finish verification. diff --git a/client/transaction-pool/graph/src/ready.rs b/client/transaction-pool/graph/src/ready.rs index 23f0d49a93071ec180822bef294888a4a6373127..c856535a6165140844a339d0539aa291236a561f 100644 --- a/client/transaction-pool/graph/src/ready.rs +++ b/client/transaction-pool/graph/src/ready.rs @@ -545,6 +545,7 @@ fn remove_item(vec: &mut Vec, item: &T) { #[cfg(test)] mod tests { use super::*; + use sp_runtime::transaction_validity::TransactionSource as Source; fn tx(id: u8) -> Transaction> { Transaction { @@ -556,6 +557,7 @@ mod tests { requires: vec![vec![1], vec![2]], provides: vec![vec![3], vec![4]], propagate: true, + source: Source::External, } } @@ -656,6 +658,7 @@ mod tests { requires: vec![tx1.provides[0].clone()], provides: vec![], propagate: true, + source: Source::External, }; // when @@ -688,6 +691,7 @@ mod tests { requires: vec![], provides: vec![], propagate: true, + source: Source::External, }; import(&mut ready, tx).unwrap(); diff --git a/client/transaction-pool/graph/src/rotator.rs b/client/transaction-pool/graph/src/rotator.rs index 55a9230522e2f7ee19e31cd004184d691ee8fd9e..be96174d1d91696832201bae8d9c6db4d1d0769f 100644 --- a/client/transaction-pool/graph/src/rotator.rs +++ b/client/transaction-pool/graph/src/rotator.rs @@ -100,6 +100,7 @@ impl PoolRotator { #[cfg(test)] mod tests { use super::*; + use sp_runtime::transaction_validity::TransactionSource; type Hash = u64; type Ex = (); @@ -122,6 +123,7 @@ mod tests { requires: vec![], provides: vec![], propagate: true, + source: TransactionSource::External, }; (hash, tx) @@ -188,6 +190,7 @@ mod tests { requires: vec![], provides: vec![], propagate: true, + source: TransactionSource::External, } } diff --git a/client/transaction-pool/graph/src/validated_pool.rs b/client/transaction-pool/graph/src/validated_pool.rs index 3a6573d9bd6c982eea42a30b36231b6822ecc416..2ff2acfe24fc8db886c2639fd8a9d2c346642532 100644 --- a/client/transaction-pool/graph/src/validated_pool.rs +++ b/client/transaction-pool/graph/src/validated_pool.rs @@ -27,15 +27,15 @@ use crate::watcher::Watcher; use serde::Serialize; use log::{debug, warn}; -use futures::channel::mpsc; use parking_lot::{Mutex, RwLock}; use sp_runtime::{ generic::BlockId, traits::{self, SaturatedConversion}, - transaction_validity::{TransactionTag as Tag, ValidTransaction}, + transaction_validity::{TransactionTag as Tag, ValidTransaction, TransactionSource}, }; use sp_transaction_pool::{error, PoolStatus}; use wasm_timer::Instant; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use crate::base_pool::PruneStatus; use crate::pool::{EventStream, Options, ChainApi, ExHash, ExtrinsicFor, TransactionFor}; @@ -58,6 +58,7 @@ impl ValidatedTransaction { pub fn valid_at( at: u64, hash: Hash, + source: TransactionSource, data: Ex, bytes: usize, validity: ValidTransaction, @@ -66,6 +67,7 @@ impl ValidatedTransaction { data, bytes, hash, + source, priority: validity.priority, requires: validity.requires, provides: validity.provides, @@ -93,7 +95,7 @@ pub struct ValidatedPool { ExHash, ExtrinsicFor, >>, - import_notification_sinks: Mutex>>>, + import_notification_sinks: Mutex>>>, rotator: PoolRotator>, } @@ -502,7 +504,7 @@ impl ValidatedPool { /// Consumers of this stream should use the `ready` method to actually get the /// pending transactions in the right order. pub fn import_notification_stream(&self) -> EventStream> { - let (sink, stream) = mpsc::unbounded(); + let (sink, stream) = tracing_unbounded("mpsc_import_notifications"); self.import_notification_sinks.lock().push(sink); stream } diff --git a/client/transaction-pool/graph/src/watcher.rs b/client/transaction-pool/graph/src/watcher.rs index d28f6814e455213a443b583646ee2af6a3abb335..d54cc2718b7cbf40f33633df249d7a8d36da826a 100644 --- a/client/transaction-pool/graph/src/watcher.rs +++ b/client/transaction-pool/graph/src/watcher.rs @@ -16,18 +16,16 @@ //! Extrinsics status updates. -use futures::{ - Stream, - channel::mpsc, -}; +use futures::Stream; use sp_transaction_pool::TransactionStatus; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; /// Extrinsic watcher. /// /// Represents a stream of status updates for particular extrinsic. #[derive(Debug)] pub struct Watcher { - receiver: mpsc::UnboundedReceiver>, + receiver: TracingUnboundedReceiver>, hash: H, } @@ -48,7 +46,7 @@ impl Watcher { /// Sender part of the watcher. Exposed only for testing purposes. #[derive(Debug)] pub struct Sender { - receivers: Vec>>, + receivers: Vec>>, is_finalized: bool, } @@ -64,7 +62,7 @@ impl Default for Sender { impl Sender { /// Add a new watcher to this sender object. pub fn new_watcher(&mut self, hash: H) -> Watcher { - let (tx, receiver) = mpsc::unbounded(); + let (tx, receiver) = tracing_unbounded("mpsc_txpool_watcher"); self.receivers.push(tx); Watcher { receiver, diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index a8888baa4839b46bddd2cb2f2b1486efc324ce33..bf104c8d78191c231b7102e97d31ccdf1237fefa 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -29,10 +29,10 @@ use sc_client_api::{ }; use sp_runtime::{ generic::BlockId, traits::{self, Block as BlockT, BlockIdTo, Header as HeaderT, Hash as HashT}, - transaction_validity::TransactionValidity, + transaction_validity::{TransactionValidity, TransactionSource}, }; use sp_transaction_pool::runtime_api::TaggedTransactionQueue; -use sp_api::ProvideRuntimeApi; +use sp_api::{ProvideRuntimeApi, ApiExt}; use crate::error::{self, Error}; @@ -81,6 +81,7 @@ impl sc_transaction_graph::ChainApi for FullChainApi, + source: TransactionSource, uxt: sc_transaction_graph::ExtrinsicFor, ) -> Self::ValidationFuture { let (tx, rx) = oneshot::channel(); @@ -88,8 +89,19 @@ impl sc_transaction_graph::ChainApi for FullChainApi, _>( + &at, |v| v >= 2, + ) + .unwrap_or_default(); + let res = if has_v2 { + runtime_api.validate_transaction(&at, source, uxt) + } else { + #[allow(deprecated)] // old validate_transaction + runtime_api.validate_transaction_before_version_2(&at, uxt) + }; + let res = res.map_err(|e| Error::RuntimeApi(format!("{:?}", e))); if let Err(e) = tx.send(res) { log::warn!("Unable to send a validate transaction result: {:?}", e); } @@ -160,6 +172,7 @@ impl sc_transaction_graph::ChainApi for LightChainApi, + source: TransactionSource, uxt: sc_transaction_graph::ExtrinsicFor, ) -> Self::ValidationFuture { let header_hash = self.client.expect_block_hash_from_id(at); @@ -174,7 +187,7 @@ impl sc_transaction_graph::ChainApi for LightChainApi BasicPool Self::with_revalidation_type(options, pool_api, RevalidationType::Full) } + /// Create new basic transaction pool with provided api, for tests. + #[cfg(test)] + pub fn new_test( + pool_api: Arc, + ) -> (Self, Pin + Send>>, intervalier::BackSignalControl) { + let pool = Arc::new(sc_transaction_graph::Pool::new(Default::default(), pool_api.clone())); + let (revalidation_queue, background_task, notifier) = + revalidation::RevalidationQueue::new_test(pool_api.clone(), pool.clone()); + ( + BasicPool { + api: pool_api, + pool, + revalidation_queue: Arc::new(revalidation_queue), + revalidation_strategy: Arc::new(Mutex::new(RevalidationStrategy::Always)), + ready_poll: Default::default(), + }, + background_task, + notifier, + ) + } + /// Create new basic transaction pool with provided api and custom /// revalidation type. pub fn with_revalidation_type( @@ -201,37 +223,40 @@ impl TransactionPool for BasicPool fn submit_at( &self, at: &BlockId, + source: TransactionSource, xts: Vec>, ) -> PoolFuture, Self::Error>>, Self::Error> { let pool = self.pool.clone(); let at = *at; async move { - pool.submit_at(&at, xts, false).await + pool.submit_at(&at, source, xts, false).await }.boxed() } fn submit_one( &self, at: &BlockId, + source: TransactionSource, xt: TransactionFor, ) -> PoolFuture, Self::Error> { let pool = self.pool.clone(); let at = *at; async move { - pool.submit_one(&at, xt).await + pool.submit_one(&at, source, xt).await }.boxed() } fn submit_and_watch( &self, at: &BlockId, + source: TransactionSource, xt: TransactionFor, ) -> PoolFuture>, Self::Error> { let at = *at; let pool = self.pool.clone(); async move { - pool.submit_and_watch(&at, xt) + pool.submit_and_watch(&at, source, xt) .map(|result| result.map(|watcher| Box::new(watcher.into_stream()) as _)) .await }.boxed() @@ -437,7 +462,14 @@ impl MaintainedTransactionPool for BasicPool resubmit_transactions.extend(block_transactions); } - if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await { + if let Err(e) = pool.submit_at( + &id, + // These transactions are coming from retracted blocks, we should + // simply consider them external. + TransactionSource::External, + resubmit_transactions, + true + ).await { log::debug!( target: "txpool", "[{:?}] Error re-submitting transactions: {:?}", id, e diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index 4b3f9955ca7a9a954debcaea19f1bf19a3bdff9b..9bcb2dac395816a73ff9799f4fe533845c06205a 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -22,10 +22,10 @@ use sc_transaction_graph::{ChainApi, Pool, ExHash, NumberFor, ValidatedTransacti use sp_runtime::traits::{Zero, SaturatedConversion}; use sp_runtime::generic::BlockId; use sp_runtime::transaction_validity::TransactionValidityError; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; -use futures::{prelude::*, channel::mpsc, stream::unfold}; +use futures::prelude::*; use std::time::Duration; -use futures_timer::Delay; #[cfg(not(test))] const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(200); @@ -53,12 +53,6 @@ struct RevalidationWorker { impl Unpin for RevalidationWorker {} -fn interval(duration: Duration) -> impl Stream + Unpin { - unfold((), move |_| { - Delay::new(duration).map(|_| Some(((), ()))) - }).map(drop) -} - /// Revalidate batch of transaction. /// /// Each transaction is validated against chain, and invalid are @@ -78,7 +72,7 @@ async fn batch_revalidate( None => continue, }; - match api.validate_transaction(&BlockId::Number(at), ext.data.clone()).await { + match api.validate_transaction(&BlockId::Number(at), ext.source, ext.data.clone()).await { Ok(Err(TransactionValidityError::Invalid(err))) => { log::debug!(target: "txpool", "[{:?}]: Revalidation: invalid {:?}", ext_hash, err); invalid_hashes.push(ext_hash); @@ -94,6 +88,7 @@ async fn batch_revalidate( ValidatedTransaction::valid_at( at.saturated_into::(), ext_hash, + ext.source, ext.data.clone(), api.hash_and_length(&ext.data).1, validity, @@ -206,8 +201,13 @@ impl RevalidationWorker { /// It does two things: periodically tries to process some transactions /// from the queue and also accepts messages to enqueue some more /// transactions from the pool. - pub async fn run(mut self, from_queue: mpsc::UnboundedReceiver>) { - let interval = interval(BACKGROUND_REVALIDATION_INTERVAL).fuse(); + pub async fn run( + mut self, + from_queue: TracingUnboundedReceiver>, + interval: R, + ) where R: Send, R::Guard: Send + { + let interval = interval.into_stream().fuse(); let from_queue = from_queue.fuse(); futures::pin_mut!(interval, from_queue); let this = &mut self; @@ -253,7 +253,7 @@ impl RevalidationWorker { pub struct RevalidationQueue { pool: Arc>, api: Arc, - background: Option>>, + background: Option>>, } impl RevalidationQueue @@ -269,11 +269,14 @@ where } } - /// New revalidation queue with background worker. - pub fn new_background(api: Arc, pool: Arc>) -> - (Self, Pin + Send>>) + pub fn new_with_interval( + api: Arc, + pool: Arc>, + interval: R, + ) -> (Self, Pin + Send>>) + where R: Send + 'static, R::Guard: Send { - let (to_worker, from_queue) = mpsc::unbounded(); + let (to_worker, from_queue) = tracing_unbounded("mpsc_revalidation_queue"); let worker = RevalidationWorker::new(api.clone(), pool.clone()); @@ -284,7 +287,25 @@ where background: Some(to_worker), }; - (queue, worker.run(from_queue).boxed()) + (queue, worker.run(from_queue, interval).boxed()) + } + + /// New revalidation queue with background worker. + pub fn new_background(api: Arc, pool: Arc>) -> + (Self, Pin + Send>>) + { + Self::new_with_interval(api, pool, intervalier::Interval::new(BACKGROUND_REVALIDATION_INTERVAL)) + } + + /// New revalidation queue with background worker and test signal. + #[cfg(test)] + pub fn new_test(api: Arc, pool: Arc>) -> + (Self, Pin + Send>>, intervalier::BackSignalControl) + { + let (interval, notifier) = intervalier::BackSignalInterval::new(BACKGROUND_REVALIDATION_INTERVAL); + let (queue, background) = Self::new_with_interval(api, pool, interval); + + (queue, background, notifier) } /// Queue some transaction for later revalidation. @@ -312,9 +333,9 @@ where #[cfg(test)] mod tests { - use super::*; use sc_transaction_graph::Pool; + use sp_transaction_pool::TransactionSource; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; use futures::executor::block_on; use substrate_test_runtime_client::{ @@ -334,7 +355,9 @@ mod tests { let queue = Arc::new(RevalidationQueue::new(api.clone(), pool.clone())); let uxt = uxt(Alice, 0); - let uxt_hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).expect("Should be valid"); + let uxt_hash = block_on( + pool.submit_one(&BlockId::number(0), TransactionSource::External, uxt.clone()) + ).expect("Should be valid"); block_on(queue.revalidate_later(0, vec![uxt_hash])); @@ -343,4 +366,4 @@ mod tests { // number of ready assert_eq!(pool.validated_pool().status().ready, 1); } -} \ No newline at end of file +} diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index 27cb6c62a53273251507d6d89162af19ec2a2701..c6c54d97209445f17b2daad50377555cdd0906e4 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -20,26 +20,32 @@ use futures::executor::block_on; use txpool::{self, Pool}; use sp_runtime::{ generic::BlockId, - transaction_validity::ValidTransaction, + transaction_validity::{ValidTransaction, InvalidTransaction, TransactionSource}, }; use substrate_test_runtime_client::{ - runtime::{Block, Hash, Index, Header, Extrinsic}, + runtime::{Block, Hash, Index, Header, Extrinsic, Transfer}, AccountKeyring::*, }; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; -use crate::revalidation::BACKGROUND_REVALIDATION_INTERVAL; -use futures::task::Poll; +use futures::{prelude::*, task::Poll}; +use codec::Encode; fn pool() -> Pool { Pool::new(Default::default(), TestApi::with_alice_nonce(209).into()) } -fn maintained_pool() -> (BasicPool, futures::executor::ThreadPool) { - let (pool, background_task) = BasicPool::new(Default::default(), std::sync::Arc::new(TestApi::with_alice_nonce(209))); +fn maintained_pool() -> ( + BasicPool, + futures::executor::ThreadPool, + intervalier::BackSignalControl, +) { + let (pool, background_task, notifier) = BasicPool::new_test( + std::sync::Arc::new(TestApi::with_alice_nonce(209)) + ); let thread_pool = futures::executor::ThreadPool::new().unwrap(); - thread_pool.spawn_ok(background_task.expect("basic pool have background task")); - (pool, thread_pool) + thread_pool.spawn_ok(background_task); + (pool, thread_pool, notifier) } fn header(number: u64) -> Header { @@ -52,10 +58,12 @@ fn header(number: u64) -> Header { } } +const SOURCE: TransactionSource = TransactionSource::External; + #[test] fn submission_should_work() { let pool = pool(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209]); @@ -64,8 +72,8 @@ fn submission_should_work() { #[test] fn multiple_submission_should_work() { let pool = pool(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 210))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); @@ -74,7 +82,7 @@ fn multiple_submission_should_work() { #[test] fn early_nonce_should_be_culled() { let pool = pool(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 208))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 208))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); @@ -84,11 +92,11 @@ fn early_nonce_should_be_culled() { fn late_nonce_should_be_queued() { let pool = pool(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 210))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); } @@ -96,8 +104,8 @@ fn late_nonce_should_be_queued() { #[test] fn prune_tags_should_work() { let pool = pool(); - let hash209 = block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); + let hash209 = block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 210))).unwrap(); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); @@ -118,16 +126,16 @@ fn prune_tags_should_work() { fn should_ban_invalid_transactions() { let pool = pool(); let uxt = uxt(Alice, 209); - let hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap(); + let hash = block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt.clone())).unwrap(); pool.validated_pool().remove_invalid(&[hash]); - block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap_err(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt.clone())).unwrap_err(); // when let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); // then - block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap_err(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt.clone())).unwrap_err(); } #[test] @@ -138,7 +146,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { })); let pool = Pool::new(Default::default(), api.clone()); let xt = uxt(Alice, 209); - block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.validated_pool().status().ready, 1); // remove the transaction that just got imported. @@ -151,7 +159,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // so now let's insert another transaction that also provides the 155 api.increment_nonce(Alice.into()); let xt = uxt(Alice, 211); - block_on(pool.submit_one(&BlockId::number(2), xt.clone())).expect("2. Imported"); + block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt.clone())).expect("2. Imported"); assert_eq!(pool.validated_pool().status().ready, 1); assert_eq!(pool.validated_pool().status().future, 1); let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); @@ -187,9 +195,9 @@ fn block_event_with_retracted(id: u64, retracted: Vec) -> ChainEvent>(), @@ -396,11 +403,11 @@ fn should_push_watchers_during_maintaince() { #[test] fn can_track_heap_size() { - let (pool, _guard) = maintained_pool(); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).expect("1. Imported"); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).expect("1. Imported"); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 211))).expect("1. Imported"); - block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 212))).expect("1. Imported"); + let (pool, _guard, _notifier) = maintained_pool(); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 210))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 211))).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 212))).expect("1. Imported"); assert!(parity_util_mem::malloc_size(&pool) > 3000); } @@ -411,7 +418,9 @@ fn finalization() { let api = TestApi::with_alice_nonce(209); api.push_block(1, vec![]); let (pool, _background) = BasicPool::new(Default::default(), api.into()); - let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + let watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone()) + ).expect("1. Imported"); pool.api.push_block(2, vec![xt.clone()]); let header = pool.api.chain().read().header_by_number.get(&2).cloned().unwrap(); @@ -461,7 +470,9 @@ fn fork_aware_finalization() { // block B1 { - let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_alice.clone())).expect("1. Imported"); + let watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, from_alice.clone()) + ).expect("1. Imported"); let header = pool.api.push_block(2, vec![from_alice.clone()]); canon_watchers.push((watcher, header.hash())); assert_eq!(pool.status().ready, 1); @@ -482,8 +493,9 @@ fn fork_aware_finalization() { // block C2 { let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]); - from_dave_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_dave.clone())) - .expect("1. Imported"); + from_dave_watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, from_dave.clone()) + ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); let event = ChainEvent::NewBlock { id: BlockId::Hash(header.hash()), @@ -498,7 +510,9 @@ fn fork_aware_finalization() { // block D2 { - from_bob_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_bob.clone())).expect("1. Imported"); + from_bob_watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, from_bob.clone()) + ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]); @@ -515,7 +529,9 @@ fn fork_aware_finalization() { // block C1 { - let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_charlie.clone())).expect("1.Imported"); + let watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, from_charlie.clone()) + ).expect("1.Imported"); assert_eq!(pool.status().ready, 1); let header = pool.api.push_block(3, vec![from_charlie.clone()]); @@ -535,7 +551,9 @@ fn fork_aware_finalization() { // block D1 { let xt = uxt(Eve, 0); - let w = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + let w = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone()) + ).expect("1. Imported"); assert_eq!(pool.status().ready, 3); let header = pool.api.push_block(4, vec![xt.clone()]); canon_watchers.push((w, header.hash())); @@ -605,21 +623,21 @@ fn fork_aware_finalization() { #[test] fn ready_set_should_not_resolve_before_block_update() { - let (pool, _guard) = maintained_pool(); + let (pool, _guard, _notifier) = maintained_pool(); let xt1 = uxt(Alice, 209); - block_on(pool.submit_one(&BlockId::number(1), xt1.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); assert!(pool.ready_at(1).now_or_never().is_none()); } #[test] fn ready_set_should_resolve_after_block_update() { - let (pool, _guard) = maintained_pool(); + let (pool, _guard, _notifier) = maintained_pool(); pool.api.push_block(1, vec![]); let xt1 = uxt(Alice, 209); - block_on(pool.submit_one(&BlockId::number(1), xt1.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); block_on(pool.maintain(block_event(1))); assert!(pool.ready_at(1).now_or_never().is_some()); @@ -627,12 +645,12 @@ fn ready_set_should_resolve_after_block_update() { #[test] fn ready_set_should_eventually_resolve_when_block_update_arrives() { - let (pool, _guard) = maintained_pool(); + let (pool, _guard, _notifier) = maintained_pool(); pool.api.push_block(1, vec![]); let xt1 = uxt(Alice, 209); - block_on(pool.submit_one(&BlockId::number(1), xt1.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); let noop_waker = futures::task::noop_waker(); let mut context = futures::task::Context::from_waker(&noop_waker); @@ -653,4 +671,37 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { assert_eq!(data.len(), 1); } } -} \ No newline at end of file +} + +#[test] +fn should_not_accept_old_signatures() { + use std::convert::TryFrom; + + let client = Arc::new(substrate_test_runtime_client::new()); + let pool = Arc::new( + BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client))).0 + ); + + let transfer = Transfer { + from: Alice.into(), + to: Bob.into(), + nonce: 0, + amount: 1, + }; + let _bytes: sp_core::sr25519::Signature = transfer.using_encoded(|e| Alice.sign(e)).into(); + + // generated with schnorrkel 0.1.1 from `_bytes` + let old_singature = sp_core::sr25519::Signature::try_from(&hex::decode( + "c427eb672e8c441c86d31f1a81b22b43102058e9ce237cabe9897ea5099ffd426cd1c6a1f4f2869c3df57901d36bedcb295657adb3a4355add86ed234eb83108" + ).expect("hex invalid")[..]).expect("signature construction failed"); + + let xt = Extrinsic::Transfer { transfer, signature: old_singature, exhaust_resources_when_not_first: false }; + + assert_matches::assert_matches!( + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())), + Err(error::Error::Pool( + sp_transaction_pool::error::Error::InvalidTransaction(InvalidTransaction::BadProof) + )), + "Should be invalid transactiono with bad proof", + ); +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6ac08309348f0140a3e59adb48d2a3146845f465..49d47d611cac3e3b0acae94b1d5fb2a26d7075e3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog]. ## Unreleased +### Changed + +* `sc_rpc::system::SystemInfo.impl_version` now returns the full version (2.0.0-alpha.2-b950f731c-x86_64-linux-gnu) instead of the short version (1.0.0) (#5271) + ## 2.0.0-alpha.4 -> 2.0.0-alpha.5 Runtime @@ -51,4 +55,4 @@ API * client/authority-discovery: Instrument code with Prometheus (#5195) * Don't include `:code` by default in storage proofs (#5179) * client/network-gossip: Merge GossipEngine and GossipEngineInner (#5042) -* Introduce `on_runtime_upgrade` (#5058) \ No newline at end of file +* Introduce `on_runtime_upgrade` (#5058) diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index b4f1b64c4b8ea3b256ca29d0906429518623177c..d1eb924d0d3c11c4b512072ee841c80b62a89d7a 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -34,8 +34,8 @@ /primitives/core/src/sandbox.rs @pepyakin # Transaction pool -/client/transaction-pool/ @tomusdrw @NikVolf -/primitives/transaction-pool/ @tomusdrw @NikVolf +/client/transaction-pool/ @NikVolf +/primitives/transaction-pool/ @NikVolf # Offchain /client/offchain/ @tomusdrw @@ -66,9 +66,14 @@ # Inflation points /frame/staking/src/inflation.rs @thiolliere -# NPoS and Governance +# NPoS and Governance and Phragmén /frame/staking/ @kianenigma /frame/elections/ @kianenigma +/frame/elections-phragmen/ @kianenigma +/primitives/phragmen/ @kianenigma + +# Fixed point arithmetic +/primitives/sp-arithmetic/ @kianenigma @thiolliere # End to end testing of substrate node /bin/node/executor/ @kianenigma @@ -85,3 +90,7 @@ # Prometheus endpoint /utils/prometheus/ @mxinden + +# CLI API +/client/cli @cecton +/client/cli-derive @cecton diff --git a/docs/CONTRIBUTING.adoc b/docs/CONTRIBUTING.adoc index cdd9809fff864cdb1b26321d76567ac0152233e5..b573aef50d874b304f3b135b4cffd08811fcfdea 100644 --- a/docs/CONTRIBUTING.adoc +++ b/docs/CONTRIBUTING.adoc @@ -25,9 +25,10 @@ Merging pull requests once CI is successful: - it does not alter any logic (e.g. comments, dependencies, docs), then it may be tagged https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial[`insubstantial`] and merged by its author once CI is complete. - it is an urgent fix with no large change to logic, then it may be merged after a non-author contributor has approved the review once CI is complete. -. Once a PR is ready for review please add the https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview[`pleasereview`] label. Generally PRs should sit with this label for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. -. If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-looksgood[`A8-looksgood`], it is ready to merge. -. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`breaksconsensus`] +. Once a PR is ready for review please add the https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview[`A0-pleasereview`] label. Generally PRs should sit with this label for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. +. If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-mergeoncegreen[`A8-mergeoncegreen`], it is ready to merge. +. PRs must be tagged with respect to _release notes_ with https://github.com/paritytech/substrate/labels/B0-silent[`B0-silent`] and `B1-..`. The former indicates that no changes should be mentioned in any release notes. The latter indicates that the changes should be reported in the corresponding release note +. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`B2-breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`B3-breaksconsensus`] . No PR should be merged until all reviews' comments are addressed. *Reviewing pull requests*: @@ -48,16 +49,19 @@ When reviewing a pull request, the end-goal is to suggest useful changes to the === Updating Polkadot as well -If your PR changes the external APIs or interfaces used by Polkadot, **a corresponding PR on Polkadot must** be submitted as well. If you tagged the PR with `breaksapi` or `breaksconsensus` this is most certainly the case, in all other cases check for it by running step 1. +**All pull requests will be checked agains either Polkadot master, or your provided Polkadot companion PR**. That is, If your PR changes the external APIs or interfaces used by Polkadot. If you tagged the PR with `breaksapi` or `breaksconsensus` this is most certainly the case, in all other cases check for it by running step 1 below. -To update a corresponding Polkadot PR: +To create a Polkadot companion PR: -0. Pull latest Polkadot master (or clone it, if you haven't yet). -1. Replace `polkadot-master` in all `Cargo.toml` with the name of the PR-branch - e.g. by running `find . -name "Cargo.toml" -exec sed -i "s/polkadot-master/PR_BRANCH/g" {}` (and to your repo, if the branch is not on mainline); Commit this change. -2. Make the changes required to pass the build again. -3. Submit all this as a PR against the Polkadot Repo, link that new PR in the existing substrate PR for reference -4. Wait for reviews on both -5. Once both PRs have been green lit, merge the Substrate PR into master and sync the changes onto the `polkadot-master` on mainline (push it). Now revert that first commit in your Polkadot PR, reverting the name back to `polkadot-master`, push and wait for the CI to confirm all is fine. Then merge the Polkadot PR. +. Pull latest Polkadot master (or clone it, if you haven't yet). +. Override your local cargo config to point to your local substrate (pointing to your WIP branch): place `paths = ["path/to/substrate"]` in `~/.cargo/config`. +. Make the changes required and build polkadot locally. +. Submit all this as a PR against the Polkadot Repo. Link to your Polkadot PR in the _description_ of your Substrate PR as "polkadot companion: [URL]" OR use the same name for your Polkdadot branch as the Substrate branch. +. Now you should see that the `check_polkadot` CI job will build your Substrate PR agains the mentioned Polkadot branch in your PR description. +. Wait for reviews on both +. Once both PRs have been green lit, they can both be merged 🍻. + +If your PR is reviewed well, but a Polkadot PR is missing, signal it with https://github.com/paritytech/substrate/labels/A7-needspolkadotpr[`A7-needspolkadotpr`] to prevent it from getting automatically merged. As there might be multiple pending PRs that might conflict with one another, a) you should not merge the substrate PR until the Polkadot PR has also been reviewed and b) both should be merged pretty quickly after another to not block others. diff --git a/docs/README.adoc b/docs/README.adoc index 8d762fee05f8789f164b0c5a263e334c28668176..51e7748b67c094194e8b1a810fe92e8cfe4d32d8 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -291,7 +291,7 @@ cargo run --release \-- \ --chain=local \ --alice \ --node-key 0000000000000000000000000000000000000000000000000000000000000001 \ - --telemetry-url ws://telemetry.polkadot.io:1024 \ + --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ --validator In the second terminal, we'll run the following to start Bob's Substrate node on a different TCP port of 30334, and with his chain database stored locally at `/tmp/bob`. We'll specify a value for the `--bootnodes` option that will connect his node to Alice's Bootnode ID on TCP port 30333: @@ -303,7 +303,7 @@ cargo run --release \-- \ --chain=local \ --bob \ --port 30334 \ - --telemetry-url ws://telemetry.polkadot.io:1024 \ + --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ --validator Additional Substrate CLI usage options are available and may be shown by running `cargo run \-- --help`. diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index 8b6dac4e094d4ecd1e5210faa6165e85929a5d38..8b242ff0e8ef149e47b86365ba14acf7abe9cc0a 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME asset management pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } # Needed for various traits. In our case, `OnFinalize`. sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. @@ -32,3 +32,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 838d21a46244bed58dc4dcc7eece139c909be293..388eb7780bd6dc7df7525d9e015c20790b0c4594 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -157,6 +157,7 @@ decl_module! { /// Issue a new class of fungible assets. There are, and will only ever be, `total` /// such assets and they'll all belong to the `origin` initially. It will have an /// identifier `AssetId` instance: this will be specified in the `Issued` event. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn issue(origin, #[compact] total: T::Balance) { let origin = ensure_signed(origin)?; @@ -170,6 +171,7 @@ decl_module! { } /// Move some assets from one holder to another. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn transfer(origin, #[compact] id: T::AssetId, target: ::Source, @@ -188,6 +190,7 @@ decl_module! { } /// Destroy any assets of `id` owned by `origin`. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn destroy(origin, #[compact] id: T::AssetId) { let origin = ensure_signed(origin)?; let balance = >::take((id, &origin)); diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index eb9c4556f5f2cd76600748f7e337688a583138a5..36e79116618e35c296590c96a1666e03bd349784 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME AURA consensus pallet" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -46,3 +46,6 @@ std = [ "sp-timestamp/std", "pallet-timestamp/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index 64c6c22eebbb68f64624b60883f78f044baed6bc..f67d4ee038ab374a5cd2c75c947f141fc58fdde2 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -11,7 +11,7 @@ description = "FRAME pallet for authority discovery" [dependencies] sp-authority-discovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/authority-discovery" } sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } @@ -39,3 +39,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 8ee4931e488c3740174706480188dfbbdd5b1857..b8f28b432ba75f14594ee50e9b6fbfeb0ee73312 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -121,6 +121,7 @@ mod tests { type ValidatorId = AuthorityId; type ValidatorIdOf = ConvertInto; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = pallet_session::PeriodicSessions; } impl pallet_session::historical::Trait for Test { diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index 105ae68d5989129e22982992fa360504b8cf02e5..fb966113d55ee67fda4dcf946487f82a9185d6a6 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/authorship" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -33,3 +33,6 @@ std = [ "sp-io/std", "sp-authorship/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index d3c1bf752aeb8a197333946e3d4d4f8a751b765f..e6249849bf40ce08844e2a562832b984af9ef7ec 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -27,7 +27,7 @@ use frame_support::traits::{FindAuthor, VerifySeal, Get}; use codec::{Encode, Decode}; use frame_system::ensure_none; use sp_runtime::traits::{Header as HeaderT, One, Zero}; -use frame_support::weights::SimpleDispatchInfo; +use frame_support::weights::{Weight, SimpleDispatchInfo, WeighData}; use sp_inherents::{InherentIdentifier, ProvideInherent, InherentData}; use sp_authorship::{INHERENT_IDENTIFIER, UnclesInherentData, InherentError}; @@ -185,7 +185,7 @@ decl_module! { pub struct Module for enum Call where origin: T::Origin { type Error = Error; - fn on_initialize(now: T::BlockNumber) { + fn on_initialize(now: T::BlockNumber) -> Weight { let uncle_generations = T::UncleGenerations::get(); // prune uncles that are older than the allowed number of generations. if uncle_generations <= now { @@ -196,6 +196,8 @@ decl_module! { ::DidSetUncles::put(false); T::EventHandler::note_author(Self::author()); + + SimpleDispatchInfo::default().weigh_data(()) } fn on_finalize() { @@ -205,7 +207,7 @@ decl_module! { } /// Provide a set of uncles. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = SimpleDispatchInfo::FixedMandatory(10_000)] fn set_uncles(origin, new_uncles: Vec) -> dispatch::DispatchResult { ensure_none(origin)?; ensure!(new_uncles.len() <= MAX_UNCLES, Error::::TooManyUncles); diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index e4acd52ed32fe1843aff5f87732d347ee20d5457..8f885500224053f5ccb8ecb80615b468a34bc514 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -9,8 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions." [dependencies] -hex-literal = "0.2.1" -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -26,11 +25,7 @@ sp-consensus-vrf = { version = "0.8.0-alpha.5", default-features = false, path = sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} [dev-dependencies] -lazy_static = "1.4.0" -parking_lot = "0.10.0" -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } [features] default = ["std"] @@ -50,3 +45,6 @@ std = [ "pallet-session/std", "sp-io/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 27a471f6f8f10f0c0d773edcb26ac1a044a5429d..5f85b910880670e646eb7c903688f754fcefa1b3 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -19,14 +19,17 @@ #![cfg_attr(not(feature = "std"), no_std)] #![forbid(unused_must_use, unsafe_code, unused_variables, unused_must_use)] -#![deny(unused_imports)] -pub use pallet_timestamp; + +use pallet_timestamp; use sp_std::{result, prelude::*}; -use frame_support::{decl_storage, decl_module, traits::{FindAuthor, Get, Randomness as RandomnessT}}; +use frame_support::{ + decl_storage, decl_module, traits::{FindAuthor, Get, Randomness as RandomnessT}, + weights::{Weight, SimpleDispatchInfo, WeighData}, +}; use sp_timestamp::OnTimestampSet; -use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill, PerThing}; -use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash}; +use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill}; +use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash, One}; use sp_staking::{ SessionIndex, offence::{Offence, Kind}, @@ -149,6 +152,13 @@ decl_storage! { /// Temporary value (cleared at block finalization) which is `Some` /// if per-block initialization has already been called for current block. Initialized get(fn initialized): Option; + + /// How late the current block is compared to its parent. + /// + /// This entry is populated as part of block execution and is cleaned up + /// on block finalization. Querying this storage entry outside of block + /// execution context should always yield zero. + Lateness get(fn lateness): T::BlockNumber; } add_extra_genesis { config(authorities): Vec<(AuthorityId, BabeAuthorityWeight)>; @@ -171,8 +181,10 @@ decl_module! { const ExpectedBlockTime: T::Moment = T::ExpectedBlockTime::get(); /// Initialization - fn on_initialize(now: T::BlockNumber) { + fn on_initialize(now: T::BlockNumber) -> Weight { Self::do_initialize(now); + + SimpleDispatchInfo::default().weigh_data(()) } /// Block finalization @@ -185,6 +197,9 @@ decl_module! { if let Some(Some(vrf_output)) = Initialized::take() { Self::deposit_vrf_output(&vrf_output); } + + // remove temporary "environment" entry from storage + Lateness::::kill(); } } } @@ -209,12 +224,7 @@ impl FindAuthor for Module { for (id, mut data) in digests.into_iter() { if id == BABE_ENGINE_ID { let pre_digest: RawPreDigest = RawPreDigest::decode(&mut data).ok()?; - return Some(match pre_digest { - RawPreDigest::Primary { authority_index, .. } => - authority_index, - RawPreDigest::Secondary { authority_index, .. } => - authority_index, - }); + return Some(pre_digest.authority_index()) } } @@ -307,12 +317,34 @@ impl Module { // epoch 0 as having started at the slot of block 1. We want to use // the same randomness and validator set as signalled in the genesis, // so we don't rotate the epoch. - now != sp_runtime::traits::One::one() && { + now != One::one() && { let diff = CurrentSlot::get().saturating_sub(Self::current_epoch_start()); diff >= T::EpochDuration::get() } } + /// Return the _best guess_ block number, at which the next epoch change is predicted to happen. + /// + /// Returns None if the prediction is in the past; This implies an error internally in the Babe + /// and should not happen under normal circumstances. + /// + /// In other word, this is only accurate if no slots are missed. Given missed slots, the slot + /// number will grow while the block number will not. Hence, the result can be interpreted as an + /// upper bound. + // -------------- IMPORTANT NOTE -------------- + // This implementation is linked to how [`should_epoch_change`] is working. This might need to + // be updated accordingly, if the underlying mechanics of slot and epochs change. + pub fn next_expected_epoch_change(now: T::BlockNumber) -> Option { + let next_slot = Self::current_epoch_start().saturating_add(T::EpochDuration::get()); + next_slot + .checked_sub(CurrentSlot::get()) + .map(|slots_remaining| { + // This is a best effort guess. Drifts in the slot/block ratio will cause errors here. + let blocks_remaining: T::BlockNumber = slots_remaining.saturated_into(); + now.saturating_add(blocks_remaining) + }) + } + /// DANGEROUS: Enact an epoch change. Should be done on every block where `should_epoch_change` has returned `true`, /// and the caller is the only caller of this function. /// @@ -324,10 +356,7 @@ impl Module { ) { // PRECONDITION: caller has done initialization and is guaranteed // by the session module to be called before this. - #[cfg(debug_assertions)] - { - assert!(Self::initialized().is_some()) - } + debug_assert!(Self::initialized().is_some()); // Update epoch index let epoch_index = EpochIndex::get() @@ -424,13 +453,21 @@ impl Module { Self::deposit_consensus(ConsensusLog::NextEpochData(next)) } - CurrentSlot::put(digest.slot_number()); + // the slot number of the current block being initialized + let current_slot = digest.slot_number(); + + // how many slots were skipped between current and last block + let lateness = current_slot.saturating_sub(CurrentSlot::get() + 1); + let lateness = T::BlockNumber::from(lateness as u32); - if let RawPreDigest::Primary { vrf_output, .. } = digest { + Lateness::::put(lateness); + CurrentSlot::put(current_slot); + + if let RawPreDigest::Primary(primary) = digest { // place the VRF output into the `Initialized` storage item // and it'll be put onto the under-construction randomness // later, once we've decided which epoch this block is in. - Some(vrf_output) + Some(primary.vrf_output) } else { None } @@ -473,6 +510,18 @@ impl OnTimestampSet for Module { fn on_timestamp_set(_moment: T::Moment) { } } +impl frame_support::traits::EstimateNextSessionRotation for Module { + fn estimate_next_session_rotation(now: T::BlockNumber) -> Option { + Self::next_expected_epoch_change(now) + } +} + +impl frame_support::traits::Lateness for Module { + fn lateness(&self) -> T::BlockNumber { + Self::lateness() + } +} + impl sp_runtime::BoundToRuntimeAppPublic for Module { type Public = AuthorityId; } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 2ec083728e82ceb8488fa9c24efb7223ae5f450e..ea802b268e399c6f71577507df49b60288e880f5 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -16,14 +16,22 @@ //! Test utilities -use super::{Trait, Module, GenesisConfig}; +use codec::Encode; +use super::{Trait, Module, GenesisConfig, CurrentSlot}; use sp_runtime::{ - traits::IdentityLookup, Perbill, testing::{Header, UintAuthorityId}, impl_opaque_keys, + Perbill, impl_opaque_keys, + testing::{Header, UintAuthorityId, Digest, DigestItem}, + traits::IdentityLookup, +}; +use frame_system::InitKind; +use frame_support::{ + impl_outer_origin, parameter_types, StorageValue, + traits::OnInitialize, + weights::Weight, }; -use sp_version::RuntimeVersion; -use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; use sp_io; use sp_core::H256; +use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof}; impl_outer_origin!{ pub enum Origin for Test where system = frame_system {} @@ -43,7 +51,6 @@ parameter_types! { pub const MinimumPeriod: u64 = 1; pub const EpochDuration: u64 = 3; pub const ExpectedBlockTime: u64 = 1; - pub const Version: RuntimeVersion = substrate_test_runtime::VERSION; pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(16); } @@ -53,7 +60,7 @@ impl frame_system::Trait for Test { type BlockNumber = u64; type Call = (); type Hash = H256; - type Version = Version; + type Version = (); type Hashing = sp_runtime::traits::BlakeTwo256; type AccountId = DummyValidatorId; type Lookup = IdentityLookup; @@ -79,11 +86,12 @@ impl pallet_session::Trait for Test { type Event = (); type ValidatorId = ::AccountId; type ShouldEndSession = Babe; - type SessionHandler = (Babe,Babe,); + type SessionHandler = (Babe,); type SessionManager = (); type ValidatorIdOf = (); type Keys = MockSessionKeys; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = Babe; } impl pallet_timestamp::Trait for Test { @@ -106,5 +114,44 @@ pub fn new_test_ext(authorities: Vec) -> sp_io::TestExternalit t.into() } +pub fn go_to_block(n: u64, s: u64) { + let pre_digest = make_pre_digest(0, s, RawVRFOutput([1; 32]), RawVRFProof([0xff; 64])); + System::initialize(&n, &Default::default(), &Default::default(), &pre_digest, InitKind::Full); + System::set_block_number(n); + if s > 1 { + CurrentSlot::put(s); + } + // includes a call into `Babe::do_initialize`. + Session::on_initialize(n); +} + +/// Slots will grow accordingly to blocks +pub fn progress_to_block(n: u64) { + let mut slot = Babe::current_slot() + 1; + for i in System::block_number()+1..=n { + go_to_block(i, slot); + slot += 1; + } +} + +pub fn make_pre_digest( + authority_index: sp_consensus_babe::AuthorityIndex, + slot_number: sp_consensus_babe::SlotNumber, + vrf_output: RawVRFOutput, + vrf_proof: RawVRFProof, +) -> Digest { + let digest_data = sp_consensus_babe::digests::RawPreDigest::Primary( + sp_consensus_babe::digests::RawPrimaryPreDigest { + authority_index, + slot_number, + vrf_output, + vrf_proof, + } + ); + let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode()); + Digest { logs: vec![log] } +} + pub type System = frame_system::Module; pub type Babe = Module; +pub type Session = pallet_session::Module; diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs index 3fcb78ae5fb56181db91ef08577a299e73389a8b..24aba100178e058112600873602218581613c3b3 100644 --- a/frame/babe/src/tests.rs +++ b/frame/babe/src/tests.rs @@ -17,10 +17,10 @@ //! Consensus extension module tests for BABE consensus. use super::*; -use mock::{new_test_ext, Babe, System}; -use sp_runtime::{traits::OnFinalize, testing::{Digest, DigestItem}}; -use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof}; +use mock::*; +use frame_support::traits::OnFinalize; use pallet_session::ShouldEndSession; +use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof}; const EMPTY_RANDOMNESS: [u8; 32] = [ 74, 25, 49, 128, 53, 97, 244, 49, @@ -29,22 +29,6 @@ const EMPTY_RANDOMNESS: [u8; 32] = [ 217, 153, 138, 37, 48, 192, 248, 0, ]; -fn make_pre_digest( - authority_index: sp_consensus_babe::AuthorityIndex, - slot_number: sp_consensus_babe::SlotNumber, - vrf_output: RawVRFOutput, - vrf_proof: RawVRFProof, -) -> Digest { - let digest_data = sp_consensus_babe::digests::RawPreDigest::Primary { - authority_index, - slot_number, - vrf_output, - vrf_proof, - }; - let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode()); - Digest { logs: vec![log] } -} - #[test] fn empty_randomness_is_correct() { let s = compute_randomness([0; RANDOMNESS_LENGTH], 0, std::iter::empty(), None); @@ -129,3 +113,24 @@ fn authority_index() { "Trivially invalid authorities are ignored") }) } + +#[test] +fn can_predict_next_epoch_change() { + new_test_ext(vec![]).execute_with(|| { + assert_eq!(::EpochDuration::get(), 3); + // this sets the genesis slot to 6; + go_to_block(1, 6); + assert_eq!(Babe::genesis_slot(), 6); + assert_eq!(Babe::current_slot(), 6); + assert_eq!(Babe::epoch_index(), 0); + + progress_to_block(5); + + assert_eq!(Babe::epoch_index(), 5 / 3); + assert_eq!(Babe::current_slot(), 10); + + // next epoch change will be at + assert_eq!(Babe::current_epoch_start(), 9); // next change will be 12, 2 slots from now + assert_eq!(Babe::next_expected_epoch_change(System::block_number()), Some(5 + 2)); + }) +} diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index 1151fffe678ca5ef712a103664c0df7c4a9d72a3..f8a59ad15476fbd0c4f69fc418c17c67be3a7506 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet to manage balances" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -35,3 +35,6 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs index d35ec7cc8508ca28bab24e2355b7900bfe8cc8c3..a6206cd84f2c118a9e99919993cb2ab41f0302a3 100644 --- a/frame/balances/src/benchmarking.rs +++ b/frame/balances/src/benchmarking.rs @@ -16,6 +16,8 @@ //! Balances pallet benchmarking. +#![cfg(feature = "runtime-benchmarks")] + use super::*; use frame_system::RawOrigin; @@ -117,3 +119,45 @@ benchmarks! { let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); }: set_balance(RawOrigin::Root, user_lookup, 0.into(), 0.into()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests_composite::{ExtBuilder, Test}; + use frame_support::assert_ok; + + #[test] + fn transfer() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer::()); + }); + } + + #[test] + fn transfer_best_case() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer_best_case::()); + }); + } + + #[test] + fn transfer_keep_alive() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer_keep_alive::()); + }); + } + + #[test] + fn transfer_set_balance() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_set_balance::()); + }); + } + + #[test] + fn transfer_set_balance_killing() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_set_balance_killing::()); + }); + } +} diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index b83530d6352eb428c1f69ade85836504579240a6..39e15b3f4f585d38b3d68752b1803782ae0b3619 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -148,14 +148,10 @@ #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(test)] -mod tests_local; -#[cfg(test)] -mod tests_composite; -#[cfg(test)] #[macro_use] mod tests; -#[cfg(feature = "runtime-benchmarks")] +mod tests_local; +mod tests_composite; mod benchmarking; use sp_std::prelude::*; diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 98c7c856bc88a3e74c56c64e77107bc57bacd500..14caf41c1eec67fc455aae0871dd2c439029380e 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -16,6 +16,21 @@ //! Macro for creating the tests for the module. +#![cfg(test)] + +#[derive(Debug)] +pub struct CallWithDispatchInfo; +impl sp_runtime::traits::Dispatchable for CallWithDispatchInfo { + type Origin = (); + type Trait = (); + type Info = frame_support::weights::DispatchInfo; + type PostInfo = frame_support::weights::PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) + -> sp_runtime::DispatchResultWithInfo { + panic!("Do not use dummy implementation for dispatch."); + } +} + #[macro_export] macro_rules! decl_tests { ($test:ty, $ext_builder:ty, $existential_deposit:expr) => { @@ -38,7 +53,7 @@ macro_rules! decl_tests { pub type System = frame_system::Module<$test>; pub type Balances = Module<$test>; - pub const CALL: &<$test as frame_system::Trait>::Call = &(); + pub const CALL: &<$test as frame_system::Trait>::Call = &$crate::tests::CallWithDispatchInfo; /// create a transaction info struct from weight. Handy to avoid building the whole struct. pub fn info_from_weight(w: Weight) -> DispatchInfo { @@ -152,14 +167,14 @@ macro_rules! decl_tests { ChargeTransactionPayment::from(1), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); assert!( as SignedExtension>::pre_dispatch( ChargeTransactionPayment::from(0), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_ok()); @@ -170,14 +185,14 @@ macro_rules! decl_tests { ChargeTransactionPayment::from(1), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); assert!( as SignedExtension>::pre_dispatch( ChargeTransactionPayment::from(0), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); }); diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 3a5c2178f88cdc2fb8a7f74bc4a78a705588edcc..59c520f4b55b3b4ef7d9318c2c17a58cacaed3e1 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -16,14 +16,20 @@ //! Test utilities -use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header}; +#![cfg(test)] + +use sp_runtime::{ + Perbill, + traits::{ConvertInto, IdentityLookup}, + testing::Header, +}; use sp_core::H256; use sp_io; use frame_support::{impl_outer_origin, parameter_types}; use frame_support::traits::Get; use frame_support::weights::{Weight, DispatchInfo}; use std::cell::RefCell; -use crate::{GenesisConfig, Module, Trait, decl_tests}; +use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; use frame_system as system; impl_outer_origin!{ @@ -52,7 +58,7 @@ impl frame_system::Trait for Test { type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = CallWithDispatchInfo; type Hash = H256; type Hashing = ::sp_runtime::traits::BlakeTwo256; type AccountId = u64; diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 861c1972127a0751dd4372fdec80cb8588d0e1f1..3a9bfb30cecd8ddd18aaa25c9bad5a4e26a4cd44 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -16,14 +16,20 @@ //! Test utilities -use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header}; +#![cfg(test)] + +use sp_runtime::{ + Perbill, + traits::{ConvertInto, IdentityLookup}, + testing::Header, +}; use sp_core::H256; use sp_io; use frame_support::{impl_outer_origin, parameter_types}; use frame_support::traits::{Get, StorageMapShim}; use frame_support::weights::{Weight, DispatchInfo}; use std::cell::RefCell; -use crate::{GenesisConfig, Module, Trait, decl_tests}; +use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; use frame_system as system; impl_outer_origin!{ @@ -52,7 +58,7 @@ impl frame_system::Trait for Test { type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = CallWithDispatchInfo; type Hash = H256; type Hashing = ::sp_runtime::traits::BlakeTwo256; type AccountId = u64; diff --git a/frame/benchmark/Cargo.toml b/frame/benchmark/Cargo.toml index ed7f313694d491b9ab9488fb9ebdcc1d587cf60c..804a28370449c46056caf5b4e97a515f6e33b9c7 100644 --- a/frame/benchmark/Cargo.toml +++ b/frame/benchmark/Cargo.toml @@ -31,3 +31,6 @@ std = [ "frame-benchmarking/std", ] runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/benchmark/src/benchmarking.rs b/frame/benchmark/src/benchmarking.rs index 29f9e8ee972a5bf95dd774eaa0cc6dda316e8de1..1e4740da2c0595b45be30c4a1dc2496176f7d157 100644 --- a/frame/benchmark/src/benchmarking.rs +++ b/frame/benchmark/src/benchmarking.rs @@ -16,6 +16,8 @@ //! Benchmarks for common FRAME Pallet operations. +#![cfg(feature = "runtime-benchmarks")] + use super::*; use frame_system::RawOrigin; diff --git a/frame/benchmark/src/lib.rs b/frame/benchmark/src/lib.rs index ec21f437d5a2cad3e5aea5cf04abb1e8107eee85..b571ffb5b9cab6272908585270a44e86efa48e6d 100644 --- a/frame/benchmark/src/lib.rs +++ b/frame/benchmark/src/lib.rs @@ -26,8 +26,7 @@ use frame_system::{self as system, ensure_signed}; use codec::{Encode, Decode}; use sp_std::prelude::Vec; -#[cfg(feature = "runtime-benchmarks")] -pub mod benchmarking; +mod benchmarking; /// Type alias for currency balance. pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -71,6 +70,7 @@ decl_module! { fn deposit_event() = default; /// Do nothing. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn do_nothing(_origin, input: u32) { if input > 0 { return Ok(()); @@ -82,6 +82,7 @@ decl_module! { /// storage database, however, the `repeat` calls will all pull from the /// storage overlay cache. You must consider this when analyzing the /// results of the benchmark. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn read_value(_origin, repeat: u32) { for _ in 0..repeat { MyValue::get(); @@ -89,6 +90,7 @@ decl_module! { } /// Put a value into a storage value. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn put_value(_origin, repeat: u32) { for r in 0..repeat { MyValue::put(r); @@ -100,6 +102,7 @@ decl_module! { /// storage database, however, the `repeat` calls will all pull from the /// storage overlay cache. You must consider this when analyzing the /// results of the benchmark. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn exists_value(_origin, repeat: u32) { for _ in 0..repeat { MyValue::exists(); @@ -107,6 +110,7 @@ decl_module! { } /// Remove a value from storage `repeat` number of times. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn remove_value(_origin, repeat: u32) { for r in 0..repeat { MyMap::remove(r); @@ -114,6 +118,7 @@ decl_module! { } /// Read a value from storage map `repeat` number of times. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn read_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::get(r); @@ -121,6 +126,7 @@ decl_module! { } /// Insert a value into a map. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn insert_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::insert(r, r); @@ -128,6 +134,7 @@ decl_module! { } /// Check is a map contains a value `repeat` number of times. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn contains_key_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::contains_key(r); @@ -135,25 +142,29 @@ decl_module! { } /// Read a value from storage `repeat` number of times. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn remove_prefix(_origin, repeat: u32) { for r in 0..repeat { MyDoubleMap::remove_prefix(r); } } - // Add user to the list. + /// Add user to the list. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn add_member_list(origin) { let who = ensure_signed(origin)?; MyMemberList::::mutate(|x| x.push(who)); } - // Append user to the list. + /// Append user to the list. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn append_member_list(origin) { let who = ensure_signed(origin)?; MyMemberList::::append(&[who])?; } - // Encode a vector of accounts to bytes. + /// Encode a vector of accounts to bytes. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn encode_accounts(_origin, accounts: Vec) { let bytes = accounts.encode(); @@ -164,7 +175,8 @@ decl_module! { } } - // Decode bytes into a vector of accounts. + /// Decode bytes into a vector of accounts. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn decode_accounts(_origin, bytes: Vec) { let accounts: Vec = Decode::decode(&mut bytes.as_slice()).map_err(|_| "Could not decode")?; diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index dcdb4f65fab48e124ae18db43cccc14d824dc9ae..3221c7a1d4d46fd0f2ef4698a3692f3fe76a800e 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -10,7 +10,8 @@ description = "Macro for benchmarking a FRAME runtime." [dependencies] linregress = "0.1" -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +paste = "0.1" +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api", default-features = false } sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../primitives/runtime-interface", default-features = false } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime", default-features = false } @@ -30,3 +31,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index f6094739bfdedf1f4fd719ec982a77566fe7348b..49f398d59d96d96ebdc1c27a2b15586acfd2fe33 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -29,6 +29,7 @@ pub use analysis::Analysis; #[doc(hidden)] pub use sp_io::storage::root as storage_root; pub use sp_runtime::traits::Dispatchable; +pub use paste; /// Construct pallet benchmarks for weighing dispatchables. /// @@ -124,6 +125,26 @@ pub use sp_runtime::traits::Dispatchable; /// }: { m.into_iter().collect::() } /// } /// ``` +/// +/// Test functions are automatically generated for each benchmark and are accessible to you when you +/// run `cargo test`. All tests are named `test_benchmark_`, expect you to pass them +/// the Runtime Trait, and run them in a test externalities environment. The test function runs your +/// benchmark just like a regular benchmark, but only testing at the lowest and highest values for +/// each component. The function will return `Ok(())` if the benchmarks return no errors. +/// +/// You can construct benchmark tests like so: +/// +/// ```ignore +/// #[test] +/// fn test_benchmarks() { +/// new_test_ext().execute_with(|| { +/// assert_ok!(test_benchmark_dummy::()); +/// assert_err!(test_benchmark_other_name::(), "Bad origin"); +/// assert_ok!(test_benchmark_sort_vector::()); +/// assert_err!(test_benchmark_broken_benchmark::(), "You forgot to sort!"); +/// }); +/// } +/// ``` #[macro_export] macro_rules! benchmarks { ( @@ -134,9 +155,31 @@ macro_rules! benchmarks { } $( $rest:tt )* ) => { - $crate::benchmarks_iter!({ - $( { $common , $common_from , $common_to , $common_instancer } )* - } ( ) $( $rest )* ); + $crate::benchmarks_iter!( + NO_INSTANCE + { $( { $common , $common_from , $common_to , $common_instancer } )* } + ( ) + $( $rest )* + ); + } +} + +#[macro_export] +macro_rules! benchmarks_instance { + ( + _ { + $( + let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr; + )* + } + $( $rest:tt )* + ) => { + $crate::benchmarks_iter!( + INSTANCE + { $( { $common , $common_from , $common_to , $common_instancer } )* } + ( ) + $( $rest )* + ); } } @@ -145,44 +188,85 @@ macro_rules! benchmarks { macro_rules! benchmarks_iter { // mutation arm: ( + $instance:ident { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) $( $rest:tt )* ) => { $crate::benchmarks_iter! { - { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) $( $rest )* + $instance + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: $name ( $origin $( , $arg )* ) + $( $rest )* } }; - // mutation arm: + // no instance mutation arm: ( + NO_INSTANCE { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) $( $rest:tt )* ) => { $crate::benchmarks_iter! { - { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { + NO_INSTANCE + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: { as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; - } $( $rest )* + } + $( $rest )* + } + }; + // instance mutation arm: + ( + INSTANCE + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + INSTANCE + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: { + as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; + } + $( $rest )* } }; // iteration arm: ( + $instance:ident { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $eval:block $( $rest:tt )* ) => { $crate::benchmark_backend! { - $name { $( $common )* } { } { $eval } { $( $code )* } + $instance + $name + { $( $common )* } + { } + { $eval } + { $( $code )* } } - $crate::benchmarks_iter!( { $( $common )* } ( $( $names )* $name ) $( $rest )* ); + $crate::benchmarks_iter!( + $instance + { $( $common )* } + ( $( $names )* $name ) + $( $rest )* + ); }; // iteration-exit arm - ( { $( $common:tt )* } ( $( $names:ident )* ) ) => { - $crate::selected_benchmark!( $( $names ),* ); - $crate::impl_benchmark!( $( $names ),* ); + ( $instance:ident { $( $common:tt )* } ( $( $names:ident )* ) ) => { + $crate::selected_benchmark!( $instance $( $names ),* ); + $crate::impl_benchmark!( $instance $( $names ),* ); + #[cfg(test)] + $crate::impl_benchmark_tests!( $( $names ),* ); } } @@ -190,7 +274,7 @@ macro_rules! benchmarks_iter { #[allow(missing_docs)] macro_rules! benchmark_backend { // parsing arms - ($name:ident { + ($instance:ident $name:ident { $( $common:tt )* } { $( PRE { $( $pre_parsed:tt )* } )* @@ -199,13 +283,13 @@ macro_rules! benchmark_backend { $( $rest:tt )* } ) => { $crate::benchmark_backend! { - $name { $( $common )* } { + $instance $name { $( $common )* } { $( PRE { $( $pre_parsed )* } )* PRE { $pre_id , $pre_ty , $pre_ex } } { $eval } { $( $rest )* } } }; - ($name:ident { + ($instance:ident $name:ident { $( $common:tt )* } { $( $parsed:tt )* @@ -214,14 +298,14 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { $( $common )* } { + $instance $name { $( $common )* } { $( $parsed )* PARAM { $param , $param_from , $param_to , $param_instancer } } { $eval } { $( $rest )* } } }; // mutation arm to look after defaulting to a common param - ($name:ident { + ($instance:ident $name:ident { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( $parsed:tt )* @@ -230,7 +314,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { + $instance $name { $( { $common , $common_from , $common_to , $common_instancer } )* } { $( $parsed )* @@ -244,7 +328,7 @@ macro_rules! benchmark_backend { } }; // mutation arm to look after defaulting only the range to common param - ($name:ident { + ($instance:ident $name:ident { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( $parsed:tt )* @@ -253,7 +337,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { + $instance $name { $( { $common , $common_from , $common_to , $common_instancer } )* } { $( $parsed )* @@ -267,7 +351,7 @@ macro_rules! benchmark_backend { } }; // mutation arm to look after a single tt for param_from. - ($name:ident { + ($instance:ident $name:ident { $( $common:tt )* } { $( $parsed:tt )* @@ -276,14 +360,14 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $param in ( $param_from ) .. $param_to => $param_instancer; $( $rest )* } } }; // mutation arm to look after the default tail of `=> ()` - ($name:ident { + ($instance:ident $name:ident { $( $common:tt )* } { $( $parsed:tt )* @@ -292,14 +376,14 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $param in $param_from .. $param_to => (); $( $rest )* } } }; // mutation arm to look after `let _ =` - ($name:ident { + ($instance:ident $name:ident { $( $common:tt )* } { $( $parsed:tt )* @@ -308,14 +392,14 @@ macro_rules! benchmark_backend { $( $rest:tt )* }) => { $crate::benchmark_backend! { - $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $pre_id : _ = $pre_ex; $( $rest )* } } }; - // actioning arm - ($name:ident { + // no instance actioning arm + (NO_INSTANCE $name:ident { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* @@ -333,6 +417,45 @@ macro_rules! benchmark_backend { ] } + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + $( + let $common = $common_from; + )* + $( + // Prepare instance + let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + )* + $( + let $pre_id : $pre_ty = $pre_ex; + )* + $( $param_instancer ; )* + $( $post )* + + Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) })) + } + } + }; + // instance actioning arm + (INSTANCE $name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* + $( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )* + } { $eval:block } { $( $post:tt )* } ) => { + #[allow(non_camel_case_types)] + struct $name; + #[allow(unused_variables)] + impl, I: Instance> $crate::BenchmarkingSetupInstance for $name { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + vec! [ + $( + ($crate::BenchmarkParameter::$param, $param_from, $param_to) + ),* + ] + } + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str> { @@ -372,7 +495,7 @@ macro_rules! benchmark_backend { #[macro_export] macro_rules! selected_benchmark { ( - $( $bench:ident ),* + NO_INSTANCE $( $bench:ident ),* ) => { // The list of available benchmarks for this pallet. #[allow(non_camel_case_types)] @@ -397,23 +520,55 @@ macro_rules! selected_benchmark { } } }; + ( + INSTANCE $( $bench:ident ),* + ) => { + // The list of available benchmarks for this pallet. + #[allow(non_camel_case_types)] + enum SelectedBenchmark { + $( $bench, )* + } + + // Allow us to select a benchmark from the list of available benchmarks. + impl, I: Instance> $crate::BenchmarkingSetupInstance for SelectedBenchmark { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetupInstance>::components(&$bench), )* + } + } + + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetupInstance>::instance(&$bench, components), )* + } + } + } + } } #[macro_export] macro_rules! impl_benchmark { ( - $( $name:ident ),* + NO_INSTANCE $( $name:ident ),* ) => { - impl $crate::Benchmarking<$crate::BenchmarkResults> for Module { + impl $crate::Benchmarking<$crate::BenchmarkResults> for Module + where T: frame_system::Trait + { + fn benchmarks() -> Vec<&'static [u8]> { + vec![ $( stringify!($name).as_ref() ),* ] + } + fn run_benchmark( - extrinsic: Vec, - lowest_range_values: Vec, - highest_range_values: Vec, - steps: Vec, + extrinsic: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], repeat: u32, ) -> Result, &'static str> { // Map the input to the selected benchmark. - let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice()) + let extrinsic = sp_std::str::from_utf8(extrinsic) .map_err(|_| "`extrinsic` is not a valid utf8 string!")?; let selected_benchmark = match extrinsic { $( stringify!($name) => SelectedBenchmark::$name, )* @@ -463,6 +618,8 @@ macro_rules! impl_benchmark { // Run the benchmark `repeat` times. for _ in 0..repeat { + // Set the block number to 1 so events are deposited. + frame_system::Module::::set_block_number(1.into()); // Set up the externalities environment for the setup we want to benchmark. let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; @@ -492,5 +649,215 @@ macro_rules! impl_benchmark { return Ok(results); } } + }; + ( + INSTANCE $( $name:ident ),* + ) => { + impl, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module + where T: frame_system::Trait + { + fn benchmarks() -> Vec<&'static [u8]> { + vec![ $( stringify!($name).as_ref() ),* ] + } + + fn run_benchmark( + extrinsic: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], + repeat: u32, + ) -> Result, &'static str> { + // Map the input to the selected benchmark. + let extrinsic = sp_std::str::from_utf8(extrinsic) + .map_err(|_| "`extrinsic` is not a valid utf8 string!")?; + let selected_benchmark = match extrinsic { + $( stringify!($name) => SelectedBenchmark::$name, )* + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + $crate::benchmarking::commit_db(); + $crate::benchmarking::wipe_db(); + + let components = >::components(&selected_benchmark); + let mut results: Vec<$crate::BenchmarkResults> = Vec::new(); + + // Default number of steps for a component. + let mut prev_steps = 10; + + // Select the component we will be benchmarking. Each component will be benchmarked. + for (idx, (name, low, high)) in components.iter().enumerate() { + // Get the number of steps for this component. + let steps = steps.get(idx).cloned().unwrap_or(prev_steps); + prev_steps = steps; + + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); + let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); + + let diff = highest - lowest; + + // Create up to `STEPS` steps for that component between high and low. + let step_size = (diff / steps).max(1); + let num_of_steps = diff / step_size + 1; + + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = lowest + step_size * s; + + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(idx, (n, _, h))| + if n == name { + (*n, component_value) + } else { + (*n, *highest_range_values.get(idx).unwrap_or(h)) + } + ) + .collect(); + + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set the block number to 1 so events are deposited. + frame_system::Module::::set_block_number(1.into()); + // Set up the externalities environment for the setup we want to benchmark. + let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + $crate::benchmarking::commit_db(); + + // Time the extrinsic logic. + let start_extrinsic = $crate::benchmarking::current_time(); + closure_to_benchmark()?; + let finish_extrinsic = $crate::benchmarking::current_time(); + let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + + // Time the storage root recalculation. + let start_storage_root = $crate::benchmarking::current_time(); + $crate::storage_root(); + let finish_storage_root = $crate::benchmarking::current_time(); + let elapsed_storage_root = finish_storage_root - start_storage_root; + + results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + + // Wipe the DB back to the genesis state. + $crate::benchmarking::wipe_db(); + } + } + } + return Ok(results); + } + } } } + +// This creates unit tests from the main benchmark macro. +// They run the benchmark using the `high` and `low` value for each component +// and ensure that everything completes successfully. +#[macro_export] +macro_rules! impl_benchmark_tests { + ( + $( $name:ident ),* + ) => { + $( + $crate::paste::item! { + fn [] () -> Result<(), &'static str> + where T: frame_system::Trait + { + let selected_benchmark = SelectedBenchmark::$name; + let components = >::components(&selected_benchmark); + + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); + + // Set the block number to 1 so events are deposited. + frame_system::Module::::set_block_number(1.into()); + // Set up the externalities environment for the setup we want to benchmark. + let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + // Run the benchmark + closure_to_benchmark()?; + // Reset the state + $crate::benchmarking::wipe_db(); + } + } + Ok(()) + } + } + )* + } +} + + +/// This macro adds pallet benchmarks to a `Vec` object. +/// +/// First create an object that holds in the input parameters for the benchmark: +/// +/// ```ignore +/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); +/// ``` +/// +/// Then define a mutable local variable to hold your `BenchmarkBatch` object: +/// +/// ```ignore +/// let mut batches = Vec::::new(); +/// ```` +/// +/// Then add the pallets you want to benchmark to this object, including the string +/// you want to use target a particular pallet: +/// +/// ```ignore +/// add_benchmark!(params, batches, b"balances", Balances); +/// add_benchmark!(params, batches, b"identity", Identity); +/// add_benchmark!(params, batches, b"session", SessionBench::); +/// ... +/// ``` +/// +/// At the end of `dispatch_benchmark`, you should return this batches object. +#[macro_export] +macro_rules! add_benchmark { + ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( + let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params; + if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { + if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { + for benchmark in $( $location )*::benchmarks().into_iter() { + $batches.push($crate::BenchmarkBatch { + results: $( $location )*::run_benchmark( + benchmark, + &lowest_range_values[..], + &highest_range_values[..], + &steps[..], + repeat, + )?, + pallet: pallet.to_vec(), + benchmark: benchmark.to_vec(), + }); + } + } else { + $batches.push($crate::BenchmarkBatch { + results: $( $location )*::run_benchmark( + &benchmark[..], + &lowest_range_values[..], + &highest_range_values[..], + &steps[..], + repeat, + )?, + pallet: pallet.to_vec(), + benchmark: benchmark.clone(), + }); + } + } + ) +} diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index 4327476c4a6c7c59dae5019935ab06879d61e353..3e79ca38751f868ee0232c9852d9b08cfe47cb05 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -22,16 +22,20 @@ use super::*; use codec::Decode; use sp_std::prelude::*; use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::{H256, Header}}; -use frame_support::{dispatch::DispatchResult, decl_module, impl_outer_origin}; +use frame_support::{ + dispatch::DispatchResult, decl_module, impl_outer_origin, assert_ok, assert_err, ensure +}; use frame_system::{RawOrigin, ensure_signed, ensure_none}; decl_module! { pub struct Module for enum Call where origin: T::Origin { + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn dummy(origin, _n: u32) -> DispatchResult { let _sender = ensure_signed(origin)?; Ok(()) } + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn other_dummy(origin, _n: u32) -> DispatchResult { let _sender = ensure_none(origin)?; Ok(()) @@ -105,13 +109,24 @@ benchmarks!{ }: other_dummy (RawOrigin::Signed(caller), b.into()) sort_vector { - let x in 0 .. 10000; + let x in 1 .. 10000; let mut m = Vec::::new(); - for i in 0..x { + for i in (0..x).rev() { m.push(i); } }: { m.sort(); + ensure!(m[0] == 0, "You forgot to sort!") + } + + broken_benchmark { + let x in 1 .. 10000; + let mut m = Vec::::new(); + for i in (0..x).rev() { + m.push(i); + } + }: { + ensure!(m[0] == 0, "You forgot to sort!") } } @@ -155,7 +170,7 @@ fn benchmarks_macro_works_for_non_dispatchable() { let selected_benchmark = SelectedBenchmark::sort_vector; let components = >::components(&selected_benchmark); - assert_eq!(components, vec![(BenchmarkParameter::x, 0, 10000)]); + assert_eq!(components, vec![(BenchmarkParameter::x, 1, 10000)]); let closure = >::instance( &selected_benchmark, @@ -164,3 +179,13 @@ fn benchmarks_macro_works_for_non_dispatchable() { assert_eq!(closure(), Ok(())); } + +#[test] +fn benchmarks_generate_unit_tests() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_dummy::()); + assert_err!(test_benchmark_other_name::(), "Bad origin"); + assert_ok!(test_benchmark_sort_vector::()); + assert_err!(test_benchmark_broken_benchmark::(), "You forgot to sort!"); + }); +} diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index bc6cfbcc86ea1e221cdceb7728fd89b2e7bb2bd9..a6f262eab99ec87f9ecd9597fb411e0772587a1d 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256; use sp_runtime::RuntimeString; /// An alphabet of possible parameters to use for benchmarking. -#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)] +#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)] #[allow(missing_docs)] #[allow(non_camel_case_types)] pub enum BenchmarkParameter { a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, } +/// The results of a single of benchmark. +#[derive(Encode, Decode, Clone, PartialEq, Debug)] +pub struct BenchmarkBatch { + /// The pallet containing this benchmark. + pub pallet: Vec, + /// The extrinsic (or benchmark name) of this benchmark. + pub benchmark: Vec, + /// The results from this benchmark. + pub results: Vec, +} + /// Results from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. @@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! { pub trait Benchmark { /// Dispatch the given benchmark. fn dispatch_benchmark( - module: Vec, - extrinsic: Vec, + pallet: Vec, + benchmark: Vec, lowest_range_values: Vec, highest_range_values: Vec, steps: Vec, repeat: u32, - ) -> Result, RuntimeString>; + ) -> Result, RuntimeString>; } } @@ -75,19 +86,24 @@ pub trait Benchmarking { /// The pallet benchmarking trait. pub trait Benchmarking { + /// Get the benchmarks available for this pallet. Generally there is one benchmark per + /// extrinsic, so these are sometimes just called "extrinsics". + fn benchmarks() -> Vec<&'static [u8]>; + /// Run the benchmarks for this pallet. /// /// Parameters - /// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes. + /// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as + /// bytes. /// - `steps`: The number of sample points you want to take across the range of parameters. /// - `lowest_range_values`: The lowest number for each range of parameters. /// - `highest_range_values`: The highest number for each range of parameters. /// - `repeat`: The number of times you want to repeat a benchmark. fn run_benchmark( - extrinsic: Vec, - lowest_range_values: Vec, - highest_range_values: Vec, - steps: Vec, + name: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], repeat: u32, ) -> Result, &'static str>; } @@ -101,6 +117,15 @@ pub trait BenchmarkingSetup { fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; } +/// The required setup for creating a benchmark. +pub trait BenchmarkingSetupInstance { + /// Return the components and their ranges which should be tested in this benchmark. + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; + + /// Set up the storage, and prepare a closure to test in a single run of the benchmark. + fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; +} + /// Grab an account, seeded by a name and index. pub fn account(name: &'static str, index: u32, seed: u32) -> AccountId { let entropy = (name, index, seed).using_encoded(blake2_256); diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index 4b037ab7c25180f494ffebf36de3f51cfd758098..113705c2c8b90c95f068971bb0f810c7b401debc 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -10,11 +10,12 @@ description = "Collective system: Members of a set of account IDs can make their [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } @@ -34,3 +35,11 @@ std = [ "sp-runtime/std", "frame-system/std", ] +runtime-benchmarks = [ + "frame-benchmarking", + "sp-runtime/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/collective/src/benchmarking.rs b/frame/collective/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..51db4ee109b06bc550759574432b3170f630e907 --- /dev/null +++ b/frame/collective/src/benchmarking.rs @@ -0,0 +1,185 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Staking pallet benchmarking. + +use super::*; + +use frame_system::RawOrigin as SystemOrigin; +use frame_benchmarking::{benchmarks_instance, account}; + +use frame_system::Module as System; +use crate::Module as Collective; + +const SEED: u32 = 0; + +benchmarks_instance! { + _{ + // User account seed. + let u in 1 .. 1000 => (); + // Old members. + let n in 1 .. 1000 => (); + // New members. + let m in 1 .. 1000 => (); + // Existing proposals. + let p in 1 .. 100 => (); + } + + set_members { + let m in ...; + let n in ...; + + // Construct `new_members`. + // It should influence timing since it will sort this vector. + let mut new_members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + new_members.push(member); + } + + // Set old members. + // We compute the difference of old and new members, so it should influence timing. + let mut old_members = vec![]; + for i in 0 .. n { + let old_member = account("old member", i, SEED); + old_members.push(old_member); + } + + let prime = Some(account("prime", 0, SEED)); + + Collective::::set_members(SystemOrigin::Root.into(), old_members, prime.clone())?; + + }: _(SystemOrigin::Root, new_members, prime) + + execute { + let u in ...; + + let caller: T::AccountId = account("caller", u, SEED); + let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + + }: _(SystemOrigin::Signed(caller), Box::new(proposal)) + + propose { + let u in ...; + + let caller: T::AccountId = account("caller", u, SEED); + let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + + let member_count = 0; + + }: _(SystemOrigin::Signed(caller), member_count, Box::new(proposal.into())) + + propose_else_branch { + let u in ...; + let p in ...; + + let caller: T::AccountId = account("caller", u, SEED); + let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + + let member_count = 3; + + // Add previous proposals. + for i in 0 .. p { + let proposal: T::Proposal = Call::::close(Default::default(), (i + 1).into()).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), member_count.clone(), Box::new(proposal.into()))?; + } + + }: propose(SystemOrigin::Signed(caller), member_count, Box::new(proposal.into())) + + vote { + let u in ...; + + let caller1: T::AccountId = account("caller1", u, SEED); + let caller2: T::AccountId = account("caller2", u, SEED); + + let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); + let proposal_hash = T::Hashing::hash_of(&proposal); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + + let member_count = 3; + Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + + let index = 0; + let approve = true; + + }: _(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + + vote_not_approve { + let u in ...; + + let caller1: T::AccountId = account("caller1", u, SEED); + let caller2: T::AccountId = account("caller2", u, SEED); + + let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); + let proposal_hash = T::Hashing::hash_of(&proposal); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + + let member_count = 3; + Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + + let index = 0; + let approve = false; + + }: vote(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + + vote_approved { + let u in ...; + + let caller1: T::AccountId = account("caller1", u, SEED); + let caller2: T::AccountId = account("caller2", u, SEED); + + let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); + let proposal_hash = T::Hashing::hash_of(&proposal); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + + let member_count = 2; + Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + + let index = 0; + let approve = true; + + }: vote(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + + close { + let u in ...; + + let caller1: T::AccountId = account("caller1", u, SEED); + let caller2: T::AccountId = account("caller2", u, SEED); + + let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); + let proposal_hash = T::Hashing::hash_of(&proposal); + + Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + let member_count = 2; + Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + + let index = 0; + let approve = true; + + let vote_end = T::MotionDuration::get() + 1u32.into(); + System::::set_block_number(vote_end); + + }: _(SystemOrigin::Signed(caller2), proposal_hash, index) +} diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 0f5a16ed7ae9a4ef61197f565a3827acb5144c55..49c1bd38914930a6e1cfb9d229b7f1e9aef0dac7 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -39,15 +39,18 @@ use sp_std::{prelude::*, result}; use sp_core::u32_trait::Value as U32; use sp_runtime::RuntimeDebug; -use sp_runtime::traits::{Hash, EnsureOrigin}; +use sp_runtime::traits::Hash; use frame_support::weights::SimpleDispatchInfo; use frame_support::{ dispatch::{Dispatchable, Parameter}, codec::{Encode, Decode}, - traits::{Get, ChangeMembers, InitializeMembers}, decl_module, decl_event, + traits::{Get, ChangeMembers, InitializeMembers, EnsureOrigin}, decl_module, decl_event, decl_storage, decl_error, ensure, }; use frame_system::{self as system, ensure_signed, ensure_root}; +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + /// Simple index type for proposal counting. pub type ProposalIndex = u32; @@ -57,12 +60,12 @@ pub type ProposalIndex = u32; /// vote exactly once, therefore also the number of votes for any given motion. pub type MemberCount = u32; -pub trait Trait: frame_system::Trait { +pub trait Trait: frame_system::Trait { /// The outer origin type. type Origin: From>; /// The outer call dispatch type. - type Proposal: Parameter + Dispatchable>::Origin>; + type Proposal: Parameter + Dispatchable>::Origin> + From>; /// The outer event type. type Event: From> + Into<::Event>; @@ -431,7 +434,7 @@ where pub struct EnsureMember(sp_std::marker::PhantomData<(AccountId, I)>); impl< O: Into, O>> + From>, - AccountId, + AccountId: Default, I, > EnsureOrigin for EnsureMember { type Success = AccountId; @@ -441,6 +444,11 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Member(Default::default())) + } } pub struct EnsureMembers(sp_std::marker::PhantomData<(N, AccountId, I)>); @@ -457,6 +465,11 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Members(N::VALUE, N::VALUE)) + } } pub struct EnsureProportionMoreThan( @@ -476,6 +489,11 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Members(1u32, 0u32)) + } } pub struct EnsureProportionAtLeast( @@ -495,6 +513,11 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Members(0u32, 0u32)) + } } #[cfg(test)] @@ -567,19 +590,20 @@ mod tests { ); fn make_ext() -> sp_io::TestExternalities { - GenesisConfig { + let mut ext: sp_io::TestExternalities = GenesisConfig { collective_Instance1: Some(collective::GenesisConfig { members: vec![1, 2, 3], phantom: Default::default(), }), collective: None, - }.build_storage().unwrap().into() + }.build_storage().unwrap().into(); + ext.execute_with(|| System::set_block_number(1)); + ext } #[test] fn motions_basic_environment_works() { make_ext().execute_with(|| { - System::set_block_number(1); assert_eq!(Collective::members(), vec![1, 2, 3]); assert_eq!(Collective::proposals(), Vec::::new()); }); @@ -592,7 +616,6 @@ mod tests { #[test] fn close_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); @@ -621,7 +644,6 @@ mod tests { #[test] fn close_with_prime_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(3))); @@ -645,7 +667,6 @@ mod tests { #[test] fn close_with_voting_prime_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(1))); @@ -670,7 +691,6 @@ mod tests { #[test] fn removal_of_old_voters_votes_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; @@ -705,7 +725,6 @@ mod tests { #[test] fn removal_of_old_voters_votes_works_with_set_members() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; @@ -740,7 +759,6 @@ mod tests { #[test] fn propose_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash = proposal.blake2_256().into(); let end = 4; @@ -770,7 +788,6 @@ mod tests { #[test] fn motions_ignoring_non_collective_proposals_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); assert_noop!( Collective::propose(Origin::signed(42), 3, Box::new(proposal.clone())), @@ -782,7 +799,6 @@ mod tests { #[test] fn motions_ignoring_non_collective_votes_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -810,7 +826,6 @@ mod tests { #[test] fn motions_revoting_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); let end = 4; @@ -862,7 +877,6 @@ mod tests { #[test] fn motions_reproposing_disapproved_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -876,7 +890,6 @@ mod tests { #[test] fn motions_disapproval_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -919,7 +932,6 @@ mod tests { #[test] fn motions_approval_works() { make_ext().execute_with(|| { - System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()))); diff --git a/frame/contracts/COMPLEXITY.md b/frame/contracts/COMPLEXITY.md index 482cb45baf993cf6ad26c912e41323d49aaa13f9..7e8c2903c79b9348d2441b46e181a0c59a6562d0 100644 --- a/frame/contracts/COMPLEXITY.md +++ b/frame/contracts/COMPLEXITY.md @@ -480,7 +480,6 @@ This paragraph concerns the following supported built-in hash functions: - `SHA2` with 256-bit width - `KECCAK` with 256-bit width - `BLAKE2` with 128-bit and 256-bit widths -- `TWOX` with 64-bit, 128-bit and 256-bit widths These functions compute a cryptographic hash on the given inputs and copy the resulting hash directly back into the sandboxed Wasm contract output buffer. diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index 3689eaa9d1d85429647a17b362190d74cb8a61b9..a9318002cee2a71d290b56687f7123d5a1f994d7 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -11,7 +11,7 @@ description = "FRAME pallet for WASM contracts" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } pwasm-utils = { version = "0.12.0", default-features = false } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } parity-wasm = { version = "0.41.0", default-features = false } wasmi-validation = { version = "0.3.0", default-features = false } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } @@ -48,3 +48,6 @@ std = [ "wasmi-validation/std", "pallet-contracts-primitives/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index 19fe69b8793466658c09605ff8a923dfc0cab481..d181896bd2a6c2bf4d25524e7fec277823f3cd02 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -10,7 +10,7 @@ description = "A crate that hosts a common definitions that are relevant for the [dependencies] # This crate should not rely on any of the frame primitives. -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } @@ -21,3 +21,6 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index 520dc4b7e5b59b6b1adc2f9469aabf02804a0c00..fa13d8ec3fe4a3fbca2f89750772a3f3a8f91251 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -9,9 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Node-specific RPC methods for interaction with contracts." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" +jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } @@ -24,3 +24,6 @@ pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.5", path = "./runtim [dev-dependencies] serde_json = "1.0.41" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index e4211dee7948951937902db87da882127043aaa3..692bd3f25ef85b3c695240897cc55bb0546eddcd 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -10,7 +10,7 @@ description = "Runtime API definition required by Contracts RPC extensions." [dependencies] sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/runtime" } pallet-contracts-primitives = { version = "2.0.0-alpha.5", default-features = false, path = "../../common" } @@ -24,3 +24,6 @@ std = [ "sp-runtime/std", "pallet-contracts-primitives/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/src/account_db.rs b/frame/contracts/src/account_db.rs index 374c55c374d7bf26c0603017f3838b9c0a6a0bb7..165581e67646df41afa60945e5baefd6e62feb30 100644 --- a/frame/contracts/src/account_db.rs +++ b/frame/contracts/src/account_db.rs @@ -146,16 +146,8 @@ impl AccountDb for DirectAccountDb { let mut total_imbalance = SignedImbalance::zero(); for (address, changed) in s.into_iter() { if let Some(balance) = changed.balance() { - let existed = !T::Currency::total_balance(&address).is_zero(); let imbalance = T::Currency::make_free_balance_be(&address, balance); - let exists = !T::Currency::total_balance(&address).is_zero(); total_imbalance = total_imbalance.merge(imbalance); - if existed && !exists { - // Account killed. This will ultimately lead to calling `OnKilledAccount` callback - // which will make removal of CodeHashOf and AccountStorage for this account. - // In order to avoid writing over the deleted properties we `continue` here. - continue; - } } if changed.code_hash().is_some() diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index eeaa35a53d278177156136df75057204f699a9bd..10938bb7debc15948c7c6af6a30db74c3dba429c 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -113,7 +113,10 @@ use sp_std::{prelude::*, marker::PhantomData, fmt::Debug}; use codec::{Codec, Encode, Decode}; use sp_io::hashing::blake2_256; use sp_runtime::{ - traits::{Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, SignedExtension}, + traits::{ + Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, SignedExtension, + DispatchInfoOf, + }, transaction_validity::{ ValidTransaction, InvalidTransaction, TransactionValidity, TransactionValidityError, }, @@ -123,9 +126,8 @@ use frame_support::dispatch::{DispatchResult, Dispatchable}; use frame_support::{ Parameter, decl_module, decl_event, decl_storage, decl_error, storage::child, parameter_types, IsSubType, - weights::DispatchInfo, }; -use frame_support::traits::{OnKilledAccount, OnUnbalanced, Currency, Get, Time, Randomness}; +use frame_support::traits::{OnUnbalanced, Currency, Get, Time, Randomness}; use frame_system::{self as system, ensure_signed, RawOrigin, ensure_root}; use sp_core::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use pallet_contracts_primitives::{RentProjection, ContractAccessError}; @@ -548,6 +550,7 @@ decl_module! { /// Updates the schedule for metering contracts. /// /// The schedule must have a greater version than the stored schedule. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn update_schedule(origin, schedule: Schedule) -> DispatchResult { ensure_root(origin)?; if >::current_schedule().version >= schedule.version { @@ -562,6 +565,7 @@ decl_module! { /// Stores the given binary Wasm code into the chain's storage and returns its `codehash`. /// You can instantiate contracts only with stored code. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn put_code( origin, #[compact] gas_limit: Gas, @@ -589,6 +593,7 @@ decl_module! { /// * If the account is a regular account, any value will be transferred. /// * If no account exists and the call value is not less than `existential_deposit`, /// a regular account will be created and any value will be transferred. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn call( origin, dest: ::Source, @@ -614,6 +619,7 @@ decl_module! { /// after the execution is saved as the `code` of the account. That code will be invoked /// upon any call received by this account. /// - The contract is initialized. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn instantiate( origin, #[compact] endowment: BalanceOf, @@ -636,6 +642,7 @@ decl_module! { /// /// If contract is not evicted as a result of this call, no actions are taken and /// the sender is not eligible for the reward. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn claim_surcharge(origin, dest: T::AccountId, aux_sender: Option) { let origin = origin.into(); let (signed, rewarded) = match (origin, aux_sender) { @@ -941,18 +948,6 @@ decl_storage! { } } -// TODO: this should be removed in favour of a self-destruct contract host function allowing the -// contract to delete all storage and the `ContractInfoOf` key and transfer remaining balance to -// some other account. As it stands, it's an economic insecurity on any smart-contract chain. -// https://github.com/paritytech/substrate/issues/4952 -impl OnKilledAccount for Module { - fn on_killed_account(who: &T::AccountId) { - if let Some(ContractInfo::Alive(info)) = >::take(who) { - child::kill_storage(&info.trie_id, info.child_trie_unique_id()); - } - } -} - /// In-memory cache of configuration values. /// /// We assume that these values can't be changed in the @@ -1098,7 +1093,6 @@ impl SignedExtension for CheckBlockGasLimit { type AccountId = T::AccountId; type Call = ::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } @@ -1107,7 +1101,7 @@ impl SignedExtension for CheckBlockGasLimit { &self, _: &Self::AccountId, call: &Self::Call, - _: Self::DispatchInfo, + _: &DispatchInfoOf, _: usize, ) -> TransactionValidity { let call = match call.is_sub_type() { diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 3f01096b883970fc50c6cb6ede5c1179658fd08b..f45b3754762f320a4906b40df156aa99696ca59c 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -117,7 +117,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnKilledAccount = Contracts; + type OnKilledAccount = (); } impl pallet_balances::Trait for Test { type Balance = u64; @@ -279,7 +279,9 @@ impl ExtBuilder { }, gas_price: self.gas_price, }.assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t) + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } } @@ -308,7 +310,7 @@ fn refunds_unused_gas() { } #[test] -fn account_removal_removes_storage() { +fn account_removal_does_not_remove_storage() { ExtBuilder::default().existential_deposit(100).build().execute_with(|| { let trie_id1 = ::TrieIdGenerator::trie_id(&1); let trie_id2 = ::TrieIdGenerator::trie_id(&2); @@ -351,14 +353,22 @@ fn account_removal_removes_storage() { // Transfer funds from account 1 of such amount that after this transfer // the balance of account 1 will be below the existential threshold. // - // This should lead to the removal of all storage associated with this account. + // This does not remove the contract storage as we are not notified about a + // account removal. This cannot happen in reality because a contract can only + // remove itself by `ext_terminate`. There is no external event that can remove + // the account appart from that. assert_ok!(Balances::transfer(Origin::signed(1), 2, 20)); - // Verify that all entries from account 1 is removed, while - // entries from account 2 is in place. + // Verify that no entries are removed. { - assert!(>::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key1).is_none()); - assert!(>::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key2).is_none()); + assert_eq!( + >::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key1), + Some(b"1".to_vec()) + ); + assert_eq!( + >::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key2), + Some(b"2".to_vec()) + ); assert_eq!( >::get_storage(&DirectAccountDb, &2, Some(&trie_id2), key1), @@ -2623,11 +2633,11 @@ fn check_block_gas_limit_works() { let call: Call = crate::Call::put_code(1000, vec![]).into(); assert_eq!( - check.validate(&0, &call, info, 0), InvalidTransaction::ExhaustsResources.into(), + check.validate(&0, &call, &info, 0), InvalidTransaction::ExhaustsResources.into(), ); let call: Call = crate::Call::update_schedule(Default::default()).into(); - assert_eq!(check.validate(&0, &call, info, 0), Ok(Default::default())); + assert_eq!(check.validate(&0, &call, &info, 0), Ok(Default::default())); }); } @@ -2747,9 +2757,6 @@ const CODE_CRYPTO_HASHES: &str = r#" (import "env" "ext_hash_keccak_256" (func $ext_hash_keccak_256 (param i32 i32 i32))) (import "env" "ext_hash_blake2_256" (func $ext_hash_blake2_256 (param i32 i32 i32))) (import "env" "ext_hash_blake2_128" (func $ext_hash_blake2_128 (param i32 i32 i32))) - (import "env" "ext_hash_twox_256" (func $ext_hash_twox_256 (param i32 i32 i32))) - (import "env" "ext_hash_twox_128" (func $ext_hash_twox_128 (param i32 i32 i32))) - (import "env" "ext_hash_twox_64" (func $ext_hash_twox_64 (param i32 i32 i32))) (import "env" "memory" (memory 1 1)) @@ -2760,9 +2767,6 @@ const CODE_CRYPTO_HASHES: &str = r#" $ext_hash_keccak_256 $ext_hash_blake2_256 $ext_hash_blake2_128 - $ext_hash_twox_256 - $ext_hash_twox_128 - $ext_hash_twox_64 ) (data (i32.const 1) "20202010201008") ;; Output sizes of the hashes in order in hex. @@ -2793,9 +2797,6 @@ const CODE_CRYPTO_HASHES: &str = r#" ;; | 1 | KECCAK | 256 | ;; | 2 | BLAKE2 | 256 | ;; | 3 | BLAKE2 | 128 | - ;; | 4 | TWOX | 256 | - ;; | 5 | TWOX | 128 | - ;; | 6 | TWOX | 64 | ;; --------------------------------- (func (export "call") (result i32) (local $chosen_hash_fn i32) @@ -2860,9 +2861,6 @@ fn crypto_hashes() { (dyn_hash_fn!(keccak_256), 32), (dyn_hash_fn!(blake2_256), 32), (dyn_hash_fn!(blake2_128), 16), - (dyn_hash_fn!(twox_256), 32), - (dyn_hash_fn!(twox_128), 16), - (dyn_hash_fn!(twox_64), 8), ]; // Test the given hash functions for the input: "_DEAD_BEEF" for (n, (hash_fn, expected_size)) in test_cases.iter().enumerate() { diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index 92f2ff782b22cdd0824ec9b2541057a123bc5df3..7cede5542fc6f8bde389f6b99fc960b7b3512e24 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -30,9 +30,6 @@ use sp_io::hashing::{ keccak_256, blake2_256, blake2_128, - twox_256, - twox_128, - twox_64, sha2_256, }; @@ -1118,78 +1115,6 @@ define_env!(Env, , ext_hash_blake2_128(ctx, input_ptr: u32, input_len: u32, output_ptr: u32) => { compute_hash_on_intermediate_buffer(ctx, blake2_128, input_ptr, input_len, output_ptr) }, - - // Computes the TWOX 256-bit hash on the given input buffer. - // - // Returns the result directly into the given output buffer. - // - // # Note - // - // - The `input` and `output` buffer may overlap. - // - The output buffer is expected to hold at least 32 bytes (256 bits). - // - It is the callers responsibility to provide an output buffer that - // is large enough to hold the expected amount of bytes returned by the - // chosen hash function. - // - // # Parameters - // - // - `input_ptr`: the pointer into the linear memory where the input - // data is placed. - // - `input_len`: the length of the input data in bytes. - // - `output_ptr`: the pointer into the linear memory where the output - // data is placed. The function will write the result - // directly into this buffer. - ext_hash_twox_256(ctx, input_ptr: u32, input_len: u32, output_ptr: u32) => { - compute_hash_on_intermediate_buffer(ctx, twox_256, input_ptr, input_len, output_ptr) - }, - - // Computes the TWOX 128-bit hash on the given input buffer. - // - // Returns the result directly into the given output buffer. - // - // # Note - // - // - The `input` and `output` buffer may overlap. - // - The output buffer is expected to hold at least 16 bytes (128 bits). - // - It is the callers responsibility to provide an output buffer that - // is large enough to hold the expected amount of bytes returned by the - // chosen hash function. - // - // # Parameters - // - // - `input_ptr`: the pointer into the linear memory where the input - // data is placed. - // - `input_len`: the length of the input data in bytes. - // - `output_ptr`: the pointer into the linear memory where the output - // data is placed. The function will write the result - // directly into this buffer. - ext_hash_twox_128(ctx, input_ptr: u32, input_len: u32, output_ptr: u32) => { - compute_hash_on_intermediate_buffer(ctx, twox_128, input_ptr, input_len, output_ptr) - }, - - // Computes the TWOX 64-bit hash on the given input buffer. - // - // Returns the result directly into the given output buffer. - // - // # Note - // - // - The `input` and `output` buffer may overlap. - // - The output buffer is expected to hold at least 8 bytes (64 bits). - // - It is the callers responsibility to provide an output buffer that - // is large enough to hold the expected amount of bytes returned by the - // chosen hash function. - // - // # Parameters - // - // - `input_ptr`: the pointer into the linear memory where the input - // data is placed. - // - `input_len`: the length of the input data in bytes. - // - `output_ptr`: the pointer into the linear memory where the output - // data is placed. The function will write the result - // directly into this buffer. - ext_hash_twox_64(ctx, input_ptr: u32, input_len: u32, output_ptr: u32) => { - compute_hash_on_intermediate_buffer(ctx, twox_64, input_ptr, input_len, output_ptr) - }, ); /// Computes the given hash function on the scratch buffer. diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index f7dd0b7f0a3c736e63f0b8891c58e469438a7aee..788645764410b9bf35dcf969e1ee152a20716a64 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -10,16 +10,18 @@ description = "FRAME pallet for democracy" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } [dev-dependencies] sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +pallet-scheduler = { version = "2.0.0-alpha.5", path = "../scheduler" } sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } hex-literal = "0.2.1" @@ -30,7 +32,16 @@ std = [ "codec/std", "sp-std/std", "sp-io/std", + "frame-benchmarking/std", "frame-support/std", "sp-runtime/std", "frame-system/std", ] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/democracy/src/benchmarking.rs b/frame/democracy/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..6165a4f8977edd32e08d1077835de8db17f30bde --- /dev/null +++ b/frame/democracy/src/benchmarking.rs @@ -0,0 +1,488 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Democracy pallet benchmarking. + +use super::*; + +use frame_benchmarking::{benchmarks, account}; +use frame_support::traits::{Currency, Get, EnsureOrigin}; +use frame_system::{RawOrigin, Module as System, self}; +use sp_runtime::traits::{Bounded, One}; + +use crate::Module as Democracy; + +const SEED: u32 = 0; +const MAX_USERS: u32 = 1000; +const MAX_REFERENDUMS: u32 = 100; +const MAX_PROPOSALS: u32 = 100; +const MAX_SECONDERS: u32 = 100; +const MAX_VETOERS: u32 = 100; +const MAX_BYTES: u32 = 16_384; + +fn funded_account(name: &'static str, index: u32) -> T::AccountId { + let caller: T::AccountId = account(name, index, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + caller +} + +fn add_proposal(n: u32) -> Result { + let other = funded_account::("proposer", n); + let value = T::MinimumDeposit::get(); + let proposal_hash: T::Hash = T::Hashing::hash_of(&n); + + Democracy::::propose(RawOrigin::Signed(other).into(), proposal_hash, value.into())?; + + Ok(proposal_hash) +} + +fn add_referendum(n: u32) -> Result { + let proposal_hash = add_proposal::(n)?; + let vote_threshold = VoteThreshold::SimpleMajority; + + Democracy::::inject_referendum( + 0.into(), + proposal_hash, + vote_threshold, + 0.into(), + ); + let referendum_index: ReferendumIndex = ReferendumCount::get() - 1; + let _ = T::Scheduler::schedule_named( + (DEMOCRACY_ID, referendum_index), + 0.into(), + None, + 63, + Call::enact_proposal(proposal_hash, referendum_index).into(), + ); + Ok(referendum_index) +} + +fn account_vote() -> AccountVote> { + let v = Vote { + aye: true, + conviction: Conviction::Locked1x, + }; + + AccountVote::Standard { + vote: v, + balance: BalanceOf::::one(), + } +} + +fn open_activate_proxy(u: u32) -> Result { + let caller = funded_account::("caller", u); + let proxy = funded_account::("proxy", u); + + Democracy::::open_proxy(RawOrigin::Signed(proxy.clone()).into(), caller.clone())?; + Democracy::::activate_proxy(RawOrigin::Signed(caller).into(), proxy.clone())?; + + Ok(proxy) +} + +benchmarks! { + _ { } + + propose { + let p in 1 .. MAX_PROPOSALS; + + // Add p proposals + for i in 0 .. p { + add_proposal::(i)?; + } + + let caller = funded_account::("caller", 0); + let proposal_hash: T::Hash = T::Hashing::hash_of(&p); + let value = T::MinimumDeposit::get(); + }: _(RawOrigin::Signed(caller), proposal_hash, value.into()) + + second { + let s in 0 .. MAX_SECONDERS; + + let caller = funded_account::("caller", 0); + let proposal_hash = add_proposal::(s)?; + + // Create s existing "seconds" + for i in 0 .. s { + let seconder = funded_account::("seconder", i); + Democracy::::second(RawOrigin::Signed(seconder).into(), 0)?; + } + + }: _(RawOrigin::Signed(caller), 0) + + vote { + let r in 1 .. MAX_REFERENDUMS; + + let caller = funded_account::("caller", 0); + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; + } + + let referendum_index = r - 1; + + }: _(RawOrigin::Signed(caller), referendum_index, account_vote) + + proxy_vote { + let r in 1 .. MAX_REFERENDUMS; + + let caller = funded_account::("caller", r); + let proxy = open_activate_proxy::(r)?; + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; + } + + let referendum_index = r - 1; + + }: _(RawOrigin::Signed(proxy), referendum_index, account_vote) + + emergency_cancel { + let u in 1 .. MAX_USERS; + + let referendum_index = add_referendum::(u)?; + let origin = T::CancellationOrigin::successful_origin(); + let call = Call::::emergency_cancel(referendum_index); + }: { + let _ = call.dispatch(origin)?; + } + + external_propose { + let u in 1 .. MAX_USERS; + + let origin = T::ExternalOrigin::successful_origin(); + let proposal_hash = T::Hashing::hash_of(&u); + let call = Call::::external_propose(proposal_hash); + }: { + let _ = call.dispatch(origin)?; + } + + external_propose_majority { + let u in 1 .. MAX_USERS; + + let origin = T::ExternalMajorityOrigin::successful_origin(); + let proposal_hash = T::Hashing::hash_of(&u); + let call = Call::::external_propose_majority(proposal_hash); + + }: { + let _ = call.dispatch(origin)?; + } + + external_propose_default { + let u in 1 .. MAX_USERS; + + let origin = T::ExternalDefaultOrigin::successful_origin(); + let proposal_hash = T::Hashing::hash_of(&u); + let call = Call::::external_propose_default(proposal_hash); + + }: { + let _ = call.dispatch(origin)?; + } + + fast_track { + let u in 1 .. MAX_USERS; + + let origin_propose = T::ExternalDefaultOrigin::successful_origin(); + let proposal_hash: T::Hash = T::Hashing::hash_of(&u); + Democracy::::external_propose_default(origin_propose, proposal_hash.clone())?; + + let origin_fast_track = T::FastTrackOrigin::successful_origin(); + let voting_period = T::FastTrackVotingPeriod::get(); + let delay = 0; + let call = Call::::fast_track(proposal_hash, voting_period.into(), delay.into()); + + }: { + let _ = call.dispatch(origin_fast_track)?; + } + + veto_external { + // Existing veto-ers + let v in 0 .. MAX_VETOERS; + + let proposal_hash: T::Hash = T::Hashing::hash_of(&v); + + let origin_propose = T::ExternalDefaultOrigin::successful_origin(); + Democracy::::external_propose_default(origin_propose, proposal_hash.clone())?; + + let mut vetoers: Vec = Vec::new(); + for i in 0 .. v { + vetoers.push(account("vetoer", i, SEED)); + } + Blacklist::::insert(proposal_hash, (T::BlockNumber::zero(), vetoers)); + + let call = Call::::veto_external(proposal_hash); + let origin = T::VetoOrigin::successful_origin(); + }: { + let _ = call.dispatch(origin)?; + } + + cancel_referendum { + let u in 1 .. MAX_USERS; + + let referendum_index = add_referendum::(u)?; + }: _(RawOrigin::Root, referendum_index) + + cancel_queued { + let u in 1 .. MAX_USERS; + + let referendum_index = add_referendum::(u)?; + }: _(RawOrigin::Root, referendum_index) + + open_proxy { + let u in 1 .. MAX_USERS; + + let caller: T::AccountId = funded_account::("caller", u); + let proxy: T::AccountId = funded_account::("proxy", u); + + }: _(RawOrigin::Signed(proxy), caller) + + activate_proxy { + let u in 1 .. MAX_USERS; + + let caller: T::AccountId = funded_account::("caller", u); + let proxy: T::AccountId = funded_account::("proxy", u); + Democracy::::open_proxy(RawOrigin::Signed(proxy.clone()).into(), caller.clone())?; + + }: _(RawOrigin::Signed(caller), proxy) + + close_proxy { + let u in 1 .. MAX_USERS; + + let proxy = open_activate_proxy::(u)?; + + }: _(RawOrigin::Signed(proxy)) + + deactivate_proxy { + let u in 1 .. MAX_USERS; + + let caller = funded_account::("caller", u); + let proxy = open_activate_proxy::(u)?; + + }: _(RawOrigin::Signed(caller), proxy) + + delegate { + let u in 1 .. MAX_USERS; + + let caller = funded_account::("caller", u); + let d: T::AccountId = funded_account::("delegate", u); + let balance = 1u32; + + }: _(RawOrigin::Signed(caller), d.into(), Conviction::Locked1x, balance.into()) + + undelegate { + let r in 1 .. MAX_REFERENDUMS; + + let other = funded_account::("other", 0); + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + } + + let delegator = funded_account::("delegator", r); + let conviction = Conviction::Locked1x; + let balance = 1u32; + + Democracy::::delegate(RawOrigin::Signed(delegator.clone()).into(), other.clone().into(), conviction, balance.into())?; + + }: _(RawOrigin::Signed(delegator)) + + clear_public_proposals { + let p in 0 .. MAX_PROPOSALS; + + for i in 0 .. p { + add_proposal::(i)?; + } + + }: _(RawOrigin::Root) + + note_preimage { + // Num of bytes in encoded proposal + let b in 0 .. MAX_BYTES; + + let caller = funded_account::("caller", b); + let encoded_proposal = vec![0; b as usize]; + }: _(RawOrigin::Signed(caller), encoded_proposal) + + note_imminent_preimage { + // Num of bytes in encoded proposal + let b in 0 .. MAX_BYTES; + + // d + 1 to include the one we are testing + let encoded_proposal = vec![0; b as usize]; + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + let block_number = T::BlockNumber::one(); + Preimages::::insert(&proposal_hash, PreimageStatus::Missing(block_number)); + + let caller = funded_account::("caller", b); + let encoded_proposal = vec![0; b as usize]; + }: _(RawOrigin::Signed(caller), encoded_proposal) + + reap_preimage { + // Num of bytes in encoded proposal + let b in 0 .. MAX_BYTES; + + let encoded_proposal = vec![0; b as usize]; + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + + let caller = funded_account::("caller", b); + Democracy::::note_preimage(RawOrigin::Signed(caller.clone()).into(), encoded_proposal.clone())?; + + // We need to set this otherwise we get `Early` error. + let block_number = T::VotingPeriod::get() + T::EnactmentPeriod::get() + T::BlockNumber::one(); + System::::set_block_number(block_number.into()); + + }: _(RawOrigin::Signed(caller), proposal_hash) + + unlock { + let u in 1 .. MAX_USERS; + + let caller = funded_account::("caller", u); + let locked_until = T::BlockNumber::zero(); + Locks::::insert(&caller, locked_until); + + T::Currency::extend_lock( + DEMOCRACY_ID, + &caller, + Bounded::max_value(), + WithdrawReason::Transfer.into() + ); + + let other = caller.clone(); + + }: _(RawOrigin::Signed(caller), other) + + remove_vote { + let r in 1 .. MAX_REFERENDUMS; + + let caller = funded_account::("caller", 0); + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; + } + + let referendum_index = r - 1; + + }: _(RawOrigin::Signed(caller), referendum_index) + + remove_other_vote { + let r in 1 .. MAX_REFERENDUMS; + + let other = funded_account::("other", r); + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + } + + let referendum_index = r - 1; + ReferendumInfoOf::::insert( + referendum_index, + ReferendumInfo::Finished { end: T::BlockNumber::zero(), approved: true } + ); + let caller = funded_account::("caller", r); + + System::::set_block_number(T::EnactmentPeriod::get() * 10u32.into()); + + }: _(RawOrigin::Signed(caller), other, referendum_index) + + proxy_delegate { + let u in 1 .. MAX_USERS; + + let other: T::AccountId = account("other", u, SEED); + let proxy = open_activate_proxy::(u)?; + let conviction = Conviction::Locked1x; + let balance = 1u32; + + }: _(RawOrigin::Signed(proxy), other, conviction, balance.into()) + + proxy_undelegate { + let r in 1 .. MAX_REFERENDUMS; + + let other = funded_account::("other", 0); + let account_vote = account_vote::(); + + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + } + + let proxy = open_activate_proxy::(r)?; + let conviction = Conviction::Locked1x; + let balance = 1u32; + Democracy::::proxy_delegate(RawOrigin::Signed(proxy.clone()).into(), other, conviction, balance.into())?; + + }: _(RawOrigin::Signed(proxy)) + + proxy_remove_vote { + let u in 1 .. MAX_USERS; + + let referendum_index = add_referendum::(u)?; + let account_vote = account_vote::(); + let proxy = open_activate_proxy::(u)?; + + Democracy::::proxy_vote(RawOrigin::Signed(proxy.clone()).into(), referendum_index, account_vote)?; + + }: _(RawOrigin::Signed(proxy), referendum_index) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_propose::()); + assert_ok!(test_benchmark_second::()); + assert_ok!(test_benchmark_vote::()); + assert_ok!(test_benchmark_proxy_vote::()); + assert_ok!(test_benchmark_emergency_cancel::()); + assert_ok!(test_benchmark_external_propose::()); + assert_ok!(test_benchmark_external_propose_majority::()); + assert_ok!(test_benchmark_external_propose_default::()); + assert_ok!(test_benchmark_fast_track::()); + assert_ok!(test_benchmark_veto_external::()); + assert_ok!(test_benchmark_cancel_referendum::()); + assert_ok!(test_benchmark_cancel_queued::()); + assert_ok!(test_benchmark_open_proxy::()); + assert_ok!(test_benchmark_activate_proxy::()); + assert_ok!(test_benchmark_close_proxy::()); + assert_ok!(test_benchmark_deactivate_proxy::()); + assert_ok!(test_benchmark_delegate::()); + assert_ok!(test_benchmark_undelegate::()); + assert_ok!(test_benchmark_clear_public_proposals::()); + assert_ok!(test_benchmark_note_preimage::()); + assert_ok!(test_benchmark_note_imminent_preimage::()); + assert_ok!(test_benchmark_reap_preimage::()); + assert_ok!(test_benchmark_unlock::()); + assert_ok!(test_benchmark_remove_vote::()); + assert_ok!(test_benchmark_remove_other_vote::()); + assert_ok!(test_benchmark_proxy_delegate::()); + assert_ok!(test_benchmark_proxy_undelegate::()); + assert_ok!(test_benchmark_proxy_remove_vote::()); + }); + } +} diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index 32371c8a3a9452e689b355df2e8f6cf42ed3d148..b09f305c642477cc277653a5ddd33be68a4a9766 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -165,15 +165,16 @@ use sp_std::prelude::*; use sp_runtime::{ - DispatchResult, DispatchError, traits::{Zero, EnsureOrigin, Hash, Dispatchable, Saturating}, + DispatchResult, DispatchError, RuntimeDebug, + traits::{Zero, Hash, Dispatchable, Saturating}, }; -use codec::{Ref, Decode}; +use codec::{Ref, Encode, Decode}; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, ensure, Parameter, - weights::SimpleDispatchInfo, + weights::{SimpleDispatchInfo, Weight, WeighData}, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, - OnUnbalanced, BalanceStatus + OnUnbalanced, BalanceStatus, schedule::Named as ScheduleNamed, EnsureOrigin } }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -190,6 +191,9 @@ pub use types::{ReferendumInfo, ReferendumStatus, ProxyState, Tally, UnvoteScope #[cfg(test)] mod tests; +#[cfg(feature = "runtime-benchmarks")] +pub mod benchmarking; + const DEMOCRACY_ID: LockIdentifier = *b"democrac"; /// A proposal index. @@ -203,7 +207,7 @@ type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; pub trait Trait: frame_system::Trait + Sized { - type Proposal: Parameter + Dispatchable; + type Proposal: Parameter + Dispatchable + From>; type Event: From> + Into<::Event>; /// Currency type for this module. @@ -270,6 +274,33 @@ pub trait Trait: frame_system::Trait + Sized { /// Handler for the unbalanced reduction when slashing a preimage deposit. type Slash: OnUnbalanced>; + + /// The Scheduler. + type Scheduler: ScheduleNamed; +} + +#[derive(Clone, Encode, Decode, RuntimeDebug)] +pub enum PreimageStatus { + /// The preimage is imminently needed at the argument. + Missing(BlockNumber), + /// The preimage is available. + Available { + data: Vec, + provider: AccountId, + deposit: Balance, + since: BlockNumber, + /// None if it's not imminent. + expiry: Option, + }, +} + +impl PreimageStatus { + fn to_missing_expiry(self) -> Option { + match self { + PreimageStatus::Missing(expiry) => Some(expiry), + _ => None, + } + } } decl_storage! { @@ -290,7 +321,7 @@ decl_storage! { // https://github.com/paritytech/substrate/issues/5322 pub Preimages: map hasher(identity) T::Hash - => Option<(Vec, T::AccountId, BalanceOf, T::BlockNumber)>; + => Option, T::BlockNumber>>; /// The next free referendum index, aka the number of referenda started so far. pub ReferendumCount get(fn referendum_count) build(|_| 0 as ReferendumIndex): ReferendumIndex; @@ -303,11 +334,6 @@ decl_storage! { map hasher(twox_64_concat) ReferendumIndex => Option>>; - // TODO: Refactor DispatchQueue into its own pallet. - // https://github.com/paritytech/substrate/issues/5322 - /// Queue of successful referenda to be dispatched. Stored ordered by block number. - pub DispatchQueue get(fn dispatch_queue): Vec<(T::BlockNumber, T::Hash, ReferendumIndex)>; - /// All votes for a particular voter. We store the balance for the number of votes that we /// have recorded. The second item is the total amount of delegations, that will be added. pub VotingOf: map hasher(twox_64_concat) T::AccountId => Voting, T::AccountId, T::BlockNumber>; @@ -499,8 +525,10 @@ decl_module! { fn deposit_event() = default; - fn on_runtime_upgrade() { + fn on_runtime_upgrade() -> Weight { Self::migrate(); + + SimpleDispatchInfo::default().weigh_data(()) } /// Propose a sensitive action to be taken. @@ -514,7 +542,8 @@ decl_module! { /// Emits `Proposed`. /// /// # - /// - `O(1)`. + /// - `O(P)` + /// - P is the number proposals in the `PublicProps` vec. /// - Two DB changes, one DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] @@ -544,7 +573,8 @@ decl_module! { /// - `proposal`: The index of the proposal to second. /// /// # - /// - `O(1)`. + /// - `O(S)`. + /// - S is the number of seconds a proposal already has. /// - One DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] @@ -566,7 +596,8 @@ decl_module! { /// - `vote`: The vote configuration. /// /// # - /// - `O(1)`. + /// - `O(R)`. + /// - R is the number of referendums the voter has voted on. /// - One DB change, one DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] @@ -608,7 +639,7 @@ decl_module! { /// -`ref_index`: The index of the referendum to cancel. /// /// # - /// - Depends on size of storage vec `VotersFor` for this referendum. + /// - `O(1)`. /// # #[weight = SimpleDispatchInfo::FixedOperational(500_000)] fn emergency_cancel(origin, ref_index: ReferendumIndex) { @@ -754,6 +785,7 @@ decl_module! { /// - One DB clear. /// - Performs a binary search on `existing_vetoers` which should not /// be very large. + /// - O(log v), v is number of `existing_vetoers` /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] fn veto_external(origin, proposal_hash: T::Hash) { @@ -802,21 +834,21 @@ decl_module! { /// /// # /// - One DB change. + /// - O(d) where d is the items in the dispatch queue. /// # #[weight = SimpleDispatchInfo::FixedOperational(10_000)] fn cancel_queued(origin, which: ReferendumIndex) { ensure_root(origin)?; - let mut items = >::get(); - let original_len = items.len(); - items.retain(|i| i.2 != which); - ensure!(items.len() < original_len, Error::::ProposalMissing); - >::put(items); + T::Scheduler::cancel_named((DEMOCRACY_ID, which)) + .map_err(|_| Error::::ProposalMissing)?; } - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { if let Err(e) = Self::begin_block(n) { sp_runtime::print(e); } + + SimpleDispatchInfo::default().weigh_data(()) } /// Specify a proxy that is already open to us. Called by the stash. @@ -974,7 +1006,14 @@ decl_module! { T::Currency::reserve(&who, deposit)?; let now = >::block_number(); - >::insert(proposal_hash, (encoded_proposal, who.clone(), deposit, now)); + let a = PreimageStatus::Available { + data: encoded_proposal, + provider: who.clone(), + deposit, + since: now, + expiry: None, + }; + >::insert(proposal_hash, a); Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, deposit)); } @@ -989,19 +1028,25 @@ decl_module! { /// Emits `PreimageNoted`. /// /// # - /// - Dependent on the size of `encoded_proposal`. + /// - Dependent on the size of `encoded_proposal` and length of dispatch queue. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] fn note_imminent_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - ensure!(!>::contains_key(&proposal_hash), Error::::DuplicatePreimage); - let queue = >::get(); - ensure!(queue.iter().any(|item| &item.1 == &proposal_hash), Error::::NotImminent); + let status = Preimages::::get(&proposal_hash).ok_or(Error::::NotImminent)?; + let expiry = status.to_missing_expiry().ok_or(Error::::DuplicatePreimage)?; let now = >::block_number(); let free = >::zero(); - >::insert(proposal_hash, (encoded_proposal, who.clone(), free, now)); + let a = PreimageStatus::Available { + data: encoded_proposal, + provider: who.clone(), + deposit: Zero::zero(), + since: now, + expiry: Some(expiry), + }; + >::insert(proposal_hash, a); Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, free)); } @@ -1024,20 +1069,22 @@ decl_module! { #[weight = SimpleDispatchInfo::FixedNormal(10_000)] fn reap_preimage(origin, proposal_hash: T::Hash) { let who = ensure_signed(origin)?; + let (provider, deposit, since, expiry) = >::get(&proposal_hash) + .and_then(|m| match m { + PreimageStatus::Available { provider, deposit, since, expiry, .. } + => Some((provider, deposit, since, expiry)), + _ => None, + }).ok_or(Error::::PreimageMissing)?; - let (_, old, deposit, then) = >::get(&proposal_hash) - .ok_or(Error::::PreimageMissing)?; let now = >::block_number(); let (voting, enactment) = (T::VotingPeriod::get(), T::EnactmentPeriod::get()); - let additional = if who == old { Zero::zero() } else { enactment }; - ensure!(now >= then + voting + additional, Error::::TooEarly); + let additional = if who == provider { Zero::zero() } else { enactment }; + ensure!(now >= since + voting + additional, Error::::TooEarly); + ensure!(expiry.map_or(true, |e| now > e), Error::::Imminent); - let queue = >::get(); - ensure!(!queue.iter().any(|item| &item.1 == &proposal_hash), Error::::Imminent); - - let _ = T::Currency::repatriate_reserved(&old, &who, deposit, BalanceStatus::Free); + let _ = T::Currency::repatriate_reserved(&provider, &who, deposit, BalanceStatus::Free); >::remove(&proposal_hash); - Self::deposit_event(RawEvent::PreimageReaped(proposal_hash, old, deposit, who)); + Self::deposit_event(RawEvent::PreimageReaped(proposal_hash, provider, deposit, who)); } /// Unlock tokens that have an expired lock. @@ -1210,6 +1257,13 @@ decl_module! { let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; Self::try_remove_vote(&target, index, UnvoteScope::Any) } + + /// Enact a proposal from a referendum. For now we just make the weight be the maximum. + #[weight = SimpleDispatchInfo::MaxNormal] + fn enact_proposal(origin, proposal_hash: T::Hash, index: ReferendumIndex) -> DispatchResult { + ensure_root(origin)?; + Self::do_enact_proposal(proposal_hash, index) + } } } @@ -1532,28 +1586,6 @@ impl Module { ref_index } - /// Enact a proposal from a referendum. - fn enact_proposal(proposal_hash: T::Hash, index: ReferendumIndex) -> DispatchResult { - if let Some((encoded_proposal, who, amount, _)) = >::take(&proposal_hash) { - if let Ok(proposal) = T::Proposal::decode(&mut &encoded_proposal[..]) { - let _ = T::Currency::unreserve(&who, amount); - Self::deposit_event(RawEvent::PreimageUsed(proposal_hash, who, amount)); - - let ok = proposal.dispatch(frame_system::RawOrigin::Root.into()).is_ok(); - Self::deposit_event(RawEvent::Executed(index, ok)); - - Ok(()) - } else { - T::Slash::on_unbalanced(T::Currency::slash_reserved(&who, amount).0); - Self::deposit_event(RawEvent::PreimageInvalid(proposal_hash, index)); - Err(Error::::PreimageInvalid.into()) - } - } else { - Self::deposit_event(RawEvent::PreimageMissing(proposal_hash, index)); - Err(Error::::PreimageMissing.into()) - } - } - /// Table the next waiting proposal for a vote. fn launch_next(now: T::BlockNumber) -> DispatchResult { if LastTabledWasExternal::take() { @@ -1610,6 +1642,28 @@ impl Module { } } + fn do_enact_proposal(proposal_hash: T::Hash, index: ReferendumIndex) -> DispatchResult { + let preimage = >::take(&proposal_hash); + if let Some(PreimageStatus::Available { data, provider, deposit, .. }) = preimage { + if let Ok(proposal) = T::Proposal::decode(&mut &data[..]) { + let _ = T::Currency::unreserve(&provider, deposit); + Self::deposit_event(RawEvent::PreimageUsed(proposal_hash, provider, deposit)); + + let ok = proposal.dispatch(frame_system::RawOrigin::Root.into()).is_ok(); + Self::deposit_event(RawEvent::Executed(index, ok)); + + Ok(()) + } else { + T::Slash::on_unbalanced(T::Currency::slash_reserved(&provider, deposit).0); + Self::deposit_event(RawEvent::PreimageInvalid(proposal_hash, index)); + Err(Error::::PreimageInvalid.into()) + } + } else { + Self::deposit_event(RawEvent::PreimageMissing(proposal_hash, index)); + Err(Error::::PreimageMissing.into()) + } + } + fn bake_referendum( now: T::BlockNumber, index: ReferendumIndex, @@ -1621,13 +1675,24 @@ impl Module { if approved { Self::deposit_event(RawEvent::Passed(index)); if status.delay.is_zero() { - let _ = Self::enact_proposal(status.proposal_hash, index); + let _ = Self::do_enact_proposal(status.proposal_hash, index); } else { - let item = (now + status.delay, status.proposal_hash, index); - >::mutate(|queue| { - let pos = queue.binary_search_by_key(&item.0, |x| x.0).unwrap_or_else(|e| e); - queue.insert(pos, item); + let when = now + status.delay; + // Note that we need the preimage now. + Preimages::::mutate_exists(&status.proposal_hash, |maybe_pre| match *maybe_pre { + Some(PreimageStatus::Available { ref mut expiry, .. }) => *expiry = Some(when), + ref mut a => *a = Some(PreimageStatus::Missing(when)), }); + + if T::Scheduler::schedule_named( + (DEMOCRACY_ID, index), + when, + None, + 63, + Call::enact_proposal(status.proposal_hash, index).into(), + ).is_err() { + frame_support::print("LOGIC ERROR: bake_referendum/schedule_named failed"); + } } } else { Self::deposit_event(RawEvent::NotPassed(index)); @@ -1650,17 +1715,6 @@ impl Module { let approved = Self::bake_referendum(now, index, info)?; ReferendumInfoOf::::insert(index, ReferendumInfo::Finished { end: now, approved }); } - - let queue = >::get(); - let mut used = 0; - // It's stored in order, so the earliest will always be at the start. - for &(_, proposal_hash, index) in queue.iter().take_while(|x| x.0 == now) { - let _ = Self::enact_proposal(proposal_hash.clone(), index); - used += 1; - } - if used != 0 { - >::put(&queue[used..]); - } Ok(()) } } diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index f2544470aa722c6186273972b083620969f98c05..e7320da08280169d5847394c02a7ab8e35637182 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -21,7 +21,7 @@ use std::cell::RefCell; use codec::Encode; use frame_support::{ impl_outer_origin, impl_outer_dispatch, assert_noop, assert_ok, parameter_types, - ord_parameter_types, traits::Contains, weights::Weight, + ord_parameter_types, traits::{Contains, OnInitialize}, weights::Weight, }; use sp_core::H256; use sp_runtime::{ @@ -71,7 +71,7 @@ impl frame_system::Trait for Test { type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = Call; type Hash = H256; type Hashing = BlakeTwo256; type AccountId = u64; @@ -90,6 +90,13 @@ impl frame_system::Trait for Test { } parameter_types! { pub const ExistentialDeposit: u64 = 1; + pub const MaximumWeight: u32 = 1000000; +} +impl pallet_scheduler::Trait for Test { + type Event = (); + type Origin = Origin; + type Call = Call; + type MaximumWeight = MaximumWeight; } impl pallet_balances::Trait for Test { type Balance = u64; @@ -119,6 +126,8 @@ impl Contains for OneToFive { fn sorted_members() -> Vec { vec![1, 2, 3, 4, 5] } + #[cfg(feature = "runtime-benchmarks")] + fn add(_m: &u64) {} } thread_local! { static PREIMAGE_BYTE_DEPOSIT: RefCell = RefCell::new(0); @@ -152,19 +161,23 @@ impl super::Trait for Test { type Slash = (); type InstantOrigin = EnsureSignedBy; type InstantAllowed = InstantAllowed; + type Scheduler = Scheduler; } -fn new_test_ext() -> sp_io::TestExternalities { +pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig::{ balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], }.assimilate_storage(&mut t).unwrap(); GenesisConfig::default().assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t) + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } type System = frame_system::Module; type Balances = pallet_balances::Module; +type Scheduler = pallet_scheduler::Module; type Democracy = Module; #[test] @@ -213,6 +226,7 @@ fn propose_set_balance_and_note(who: u64, value: u64, delay: u64) -> DispatchRes fn next_block() { System::set_block_number(System::block_number() + 1); + Scheduler::on_initialize(System::block_number()); assert_eq!(Democracy::begin_block(System::block_number()), Ok(())); } diff --git a/frame/democracy/src/tests/cancellation.rs b/frame/democracy/src/tests/cancellation.rs index c0e1b8b27ae40498482f091466a53c6d036b07d2..998b0c14d8c8074738f847b8d7b9497e3340b4b5 100644 --- a/frame/democracy/src/tests/cancellation.rs +++ b/frame/democracy/src/tests/cancellation.rs @@ -21,7 +21,6 @@ use super::*; #[test] fn cancel_referendum_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), @@ -51,13 +50,11 @@ fn cancel_queued_should_work() { fast_forward_to(4); - assert_eq!(Democracy::dispatch_queue(), vec![ - (6, set_balance_proposal_hash_and_note(2), 0) - ]); + assert!(pallet_scheduler::Agenda::::get(6)[0].is_some()); assert_noop!(Democracy::cancel_queued(Origin::ROOT, 1), Error::::ProposalMissing); assert_ok!(Democracy::cancel_queued(Origin::ROOT, 0)); - assert_eq!(Democracy::dispatch_queue(), vec![]); + assert!(pallet_scheduler::Agenda::::get(6)[0].is_none()); }); } diff --git a/frame/democracy/src/tests/lock_voting.rs b/frame/democracy/src/tests/lock_voting.rs index c46bb47b893c35769b849b9ae843a2442c24d9d9..e83d974a8dc27fc990950ed1ae3895946b643e03 100644 --- a/frame/democracy/src/tests/lock_voting.rs +++ b/frame/democracy/src/tests/lock_voting.rs @@ -140,7 +140,6 @@ fn no_locks_without_conviction_should_work() { #[test] fn lock_voting_should_work_with_delegation() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), diff --git a/frame/democracy/src/tests/preimage.rs b/frame/democracy/src/tests/preimage.rs index 1fb805f72630d5f53f8c5620ba00cc8087a697e8..7d977b0ba83acd514e73fb77d18c485269451161 100644 --- a/frame/democracy/src/tests/preimage.rs +++ b/frame/democracy/src/tests/preimage.rs @@ -21,7 +21,6 @@ use super::*; #[test] fn missing_preimage_should_fail() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash(2), @@ -40,7 +39,6 @@ fn missing_preimage_should_fail() { #[test] fn preimage_deposit_should_be_required_and_returned() { new_test_ext().execute_with(|| { - System::set_block_number(1); // fee of 100 is too much. PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 100); assert_noop!( @@ -71,7 +69,6 @@ fn preimage_deposit_should_be_required_and_returned() { #[test] fn preimage_deposit_should_be_reapable_earlier_by_owner() { new_test_ext().execute_with(|| { - System::set_block_number(1); PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); assert_ok!(Democracy::note_preimage(Origin::signed(6), set_balance_proposal(2))); @@ -93,7 +90,6 @@ fn preimage_deposit_should_be_reapable_earlier_by_owner() { #[test] fn preimage_deposit_should_be_reapable() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_noop!( Democracy::reap_preimage(Origin::signed(5), set_balance_proposal_hash(2)), Error::::PreimageMissing @@ -122,7 +118,6 @@ fn preimage_deposit_should_be_reapable() { #[test] fn noting_imminent_preimage_for_free_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); let r = Democracy::inject_referendum( @@ -152,13 +147,11 @@ fn noting_imminent_preimage_for_free_should_work() { #[test] fn reaping_imminent_preimage_should_fail() { new_test_ext().execute_with(|| { - System::set_block_number(1); let h = set_balance_proposal_hash_and_note(2); let r = Democracy::inject_referendum(3, h, VoteThreshold::SuperMajorityApprove, 1); assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); next_block(); next_block(); - // now imminent. assert_noop!(Democracy::reap_preimage(Origin::signed(6), h), Error::::Imminent); }); } diff --git a/frame/democracy/src/tests/public_proposals.rs b/frame/democracy/src/tests/public_proposals.rs index ef687146015aad4b4c4058c98dd320fcd0f573a4..04246e86f1da4fb530fb47a58bb9fbb07120bc43 100644 --- a/frame/democracy/src/tests/public_proposals.rs +++ b/frame/democracy/src/tests/public_proposals.rs @@ -21,7 +21,6 @@ use super::*; #[test] fn backing_for_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_ok!(propose_set_balance_and_note(1, 2, 2)); assert_ok!(propose_set_balance_and_note(1, 4, 4)); assert_ok!(propose_set_balance_and_note(1, 3, 3)); @@ -34,7 +33,6 @@ fn backing_for_should_work() { #[test] fn deposit_for_proposals_should_be_taken() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_ok!(propose_set_balance_and_note(1, 2, 5)); assert_ok!(Democracy::second(Origin::signed(2), 0)); assert_ok!(Democracy::second(Origin::signed(5), 0)); @@ -49,7 +47,6 @@ fn deposit_for_proposals_should_be_taken() { #[test] fn deposit_for_proposals_should_be_returned() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_ok!(propose_set_balance_and_note(1, 2, 5)); assert_ok!(Democracy::second(Origin::signed(2), 0)); assert_ok!(Democracy::second(Origin::signed(5), 0)); @@ -65,7 +62,6 @@ fn deposit_for_proposals_should_be_returned() { #[test] fn proposal_with_deposit_below_minimum_should_not_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_noop!(propose_set_balance(1, 2, 0), Error::::ValueLow); }); } @@ -73,7 +69,6 @@ fn proposal_with_deposit_below_minimum_should_not_work() { #[test] fn poor_proposer_should_not_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_noop!(propose_set_balance(1, 2, 11), BalancesError::::InsufficientBalance); }); } @@ -81,7 +76,6 @@ fn poor_proposer_should_not_work() { #[test] fn poor_seconder_should_not_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); assert_ok!(propose_set_balance_and_note(2, 2, 11)); assert_noop!(Democracy::second(Origin::signed(1), 0), BalancesError::::InsufficientBalance); }); diff --git a/frame/democracy/src/tests/scheduling.rs b/frame/democracy/src/tests/scheduling.rs index 81120ea5b56c55fbdd6df22aae36e332c1564405..db9724deddc223d8895ce807f91ce65e9c020a8b 100644 --- a/frame/democracy/src/tests/scheduling.rs +++ b/frame/democracy/src/tests/scheduling.rs @@ -21,7 +21,6 @@ use super::*; #[test] fn simple_passing_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), @@ -39,7 +38,6 @@ fn simple_passing_should_work() { #[test] fn simple_failing_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), @@ -59,7 +57,6 @@ fn simple_failing_should_work() { #[test] fn ooo_inject_referendums_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r1 = Democracy::inject_referendum( 3, set_balance_proposal_hash_and_note(3), @@ -90,7 +87,6 @@ fn ooo_inject_referendums_should_work() { #[test] fn delayed_enactment_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), diff --git a/frame/democracy/src/tests/voting.rs b/frame/democracy/src/tests/voting.rs index 06fde99cbdb5c0344bc79f54932b764e4f821879..43aed29a32d8bc8c0cc59ab0d84b7fcef44dd4aa 100644 --- a/frame/democracy/src/tests/voting.rs +++ b/frame/democracy/src/tests/voting.rs @@ -85,9 +85,7 @@ fn single_proposal_should_work() { fast_forward_to(4); assert!(Democracy::referendum_status(0).is_err()); - assert_eq!(Democracy::dispatch_queue(), vec![ - (6, set_balance_proposal_hash_and_note(2), 0) - ]); + assert!(pallet_scheduler::Agenda::::get(6)[0].is_some()); // referendum passes and wait another two blocks for enactment. fast_forward_to(6); @@ -99,7 +97,6 @@ fn single_proposal_should_work() { #[test] fn controversial_voting_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), @@ -126,7 +123,6 @@ fn controversial_voting_should_work() { #[test] fn controversial_low_turnout_voting_should_work() { new_test_ext().execute_with(|| { - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), @@ -151,7 +147,6 @@ fn passing_low_turnout_voting_should_work() { assert_eq!(Balances::free_balance(42), 0); assert_eq!(Balances::total_issuance(), 210); - System::set_block_number(1); let r = Democracy::inject_referendum( 2, set_balance_proposal_hash_and_note(2), diff --git a/frame/democracy/src/types.rs b/frame/democracy/src/types.rs index 7a145701e99f1a5c201e2d221de30a9699e3fca6..3454326364de60412048467b8602df796110e3fd 100644 --- a/frame/democracy/src/types.rs +++ b/frame/democracy/src/types.rs @@ -62,6 +62,13 @@ impl Saturating for Delegations { capital: self.capital.saturating_mul(o.capital), } } + + fn saturating_pow(self, exp: usize) -> Self { + Self { + votes: self.votes.saturating_pow(exp), + capital: self.capital.saturating_pow(exp), + } + } } impl< diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index 97b7dc124314d1869c5ec117281f04989906af37..fb219bbebc33d5a864e3d0beee08e32e07e2eab5 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -9,7 +9,8 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME election pallet for PHRAGMEN" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +serde = { version = "1.0.101", optional = true } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } sp-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/phragmen" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } @@ -20,13 +21,14 @@ sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../pr sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } hex-literal = "0.2.1" pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +pallet-scheduler = { version = "2.0.0-alpha.5", path = "../scheduler" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } -serde = { version = "1.0.101" } [features] default = ["std"] std = [ + "serde", "codec/std", "frame-support/std", "sp-runtime/std", @@ -35,3 +37,6 @@ std = [ "sp-std/std", ] runtime-benchmarks = ["frame-support/runtime-benchmarks"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index d74fb4bdcd94411a2ce87539d5f1ad3e248f7446..036a5f492c19c94c9f4ca980bb1de0846074bb37 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -87,13 +87,14 @@ use sp_runtime::{ print, DispatchResult, DispatchError, Perbill, traits::{Zero, StaticLookup, Convert}, }; use frame_support::{ - decl_storage, decl_event, ensure, decl_module, decl_error, weights::SimpleDispatchInfo, - storage::{StorageMap, IterableStorageMap}, traits::{ + decl_storage, decl_event, ensure, decl_module, decl_error, + weights::{SimpleDispatchInfo, Weight, WeighData}, storage::{StorageMap, IterableStorageMap}, + traits::{ Currency, Get, LockableCurrency, LockIdentifier, ReservableCurrency, WithdrawReasons, - ChangeMembers, OnUnbalanced, WithdrawReason, Contains, BalanceStatus + ChangeMembers, OnUnbalanced, WithdrawReason, Contains, BalanceStatus, InitializeMembers, } }; -use sp_phragmen::ExtendedBalance; +use sp_phragmen::{build_support_map, ExtendedBalance}; use frame_system::{self as system, ensure_signed, ensure_root}; const MODULE_ID: LockIdentifier = *b"phrelect"; @@ -117,6 +118,9 @@ pub trait Trait: frame_system::Trait { /// What to do when the members change. type ChangeMembers: ChangeMembers; + /// What to do with genesis members + type InitializeMembers: InitializeMembers; + /// Convert a balance into a number used for election calculation. /// This must fit into a `u64` but is allowed to be sensibly lossy. type CurrencyToVote: Convert, u64> + Convert>; @@ -159,11 +163,48 @@ decl_storage! { pub ElectionRounds get(fn election_rounds): u32 = Zero::zero(); /// Votes and locked stake of a particular voter. - pub Voting: map hasher(twox_64_concat) T::AccountId => (BalanceOf, Vec); + pub Voting get(fn voting): map hasher(twox_64_concat) T::AccountId => (BalanceOf, Vec); /// The present candidate list. Sorted based on account-id. A current member or runner-up /// can never enter this vector and is always implicitly assumed to be a candidate. pub Candidates get(fn candidates): Vec; + } add_extra_genesis { + config(members): Vec<(T::AccountId, BalanceOf)>; + build(|config: &GenesisConfig| { + let members = config.members.iter().map(|(ref member, ref stake)| { + // make sure they have enough stake + assert!( + T::Currency::free_balance(member) >= *stake, + "Genesis member does not have enough stake", + ); + + // reserve candidacy bond and set as members. + T::Currency::reserve(&member, T::CandidacyBond::get()) + .expect("Genesis member does not have enough balance to be a candidate"); + + // Note: all members will only vote for themselves, hence they must be given exactly + // their own stake as total backing. Any sane election should behave as such. + // Nonetheless, stakes will be updated for term 1 onwards according to the election. + Members::::mutate(|members| { + match members.binary_search_by(|(a, _b)| a.cmp(member)) { + Ok(_) => panic!("Duplicate member in elections phragmen genesis: {}", member), + Err(pos) => members.insert(pos, (member.clone(), *stake)), + } + }); + + // set self-votes to make persistent. + >::vote( + T::Origin::from(Some(member.clone()).into()), + vec![member.clone()], + *stake, + ).expect("Genesis member could not vote."); + + member.clone() + }).collect::>(); + + // report genesis members to upstream, if any. + T::InitializeMembers::initialize_members(&members); + }) } } @@ -223,8 +264,10 @@ decl_module! { fn deposit_event() = default; - fn on_runtime_upgrade() { + fn on_runtime_upgrade() -> Weight { migration::migrate::(); + + SimpleDispatchInfo::default().weigh_data(()) } const CandidacyBond: BalanceOf = T::CandidacyBond::get(); @@ -461,11 +504,13 @@ decl_module! { } /// What to do at the end of each block. Checks if an election needs to happen or not. - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { if let Err(e) = Self::end_block(n) { print("Guru meditation"); print(e); } + + SimpleDispatchInfo::default().weigh_data(()) } } } @@ -687,12 +732,18 @@ impl Module { .filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } ) .collect::>(); - let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>( - &new_set, - &phragmen_result.assignments, - Self::locked_stake_of, + let stake_of = |who: &T::AccountId| -> ExtendedBalance { + , u64>>::convert( + Self::locked_stake_of(who) + ) as ExtendedBalance + }; + let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + phragmen_result.assignments, + stake_of, ); + let (support_map, _) = build_support_map::(&new_set, &staked_assignments); + let to_balance = |e: ExtendedBalance| >>::convert(e); let new_set_with_stake = new_set @@ -712,7 +763,7 @@ impl Module { let mut new_members = (&new_set_with_stake[..split_point]).to_vec(); // save the runners up as-is. They are sorted based on desirability. - // sort and save the members. + // save the members, sorted based on account id. new_members.sort_by(|i, j| i.0.cmp(&j.0)); let mut prime_votes: Vec<_> = new_members.iter().map(|c| (&c.0, BalanceOf::::zero())).collect(); @@ -833,7 +884,7 @@ mod tests { Perbill, testing::Header, BuildStorage, traits::{BlakeTwo256, IdentityLookup, Block as BlockT}, }; - use crate as elections; + use crate as elections_phragmen; use frame_system as system; parameter_types! { @@ -969,6 +1020,7 @@ mod tests { type Currency = Balances; type CurrencyToVote = CurrencyToVoteHandler; type ChangeMembers = TestChangeMembers; + type InitializeMembers = (); type CandidacyBond = CandidacyBond; type VotingBond = VotingBond; type TermDuration = TermDuration; @@ -990,11 +1042,12 @@ mod tests { { System: system::{Module, Call, Event}, Balances: pallet_balances::{Module, Call, Event, Config}, - Elections: elections::{Module, Call, Event}, + Elections: elections_phragmen::{Module, Call, Event, Config}, } ); pub struct ExtBuilder { + genesis_members: Vec<(u64, u64)>, balance_factor: u64, voter_bond: u64, term_duration: u64, @@ -1004,6 +1057,7 @@ mod tests { impl Default for ExtBuilder { fn default() -> Self { Self { + genesis_members: vec![], balance_factor: 1, voter_bond: 2, desired_runners_up: 0, @@ -1025,11 +1079,16 @@ mod tests { self.term_duration = duration; self } + pub fn genesis_members(mut self, members: Vec<(u64, u64)>) -> Self { + self.genesis_members = members; + self + } pub fn build(self) -> sp_io::TestExternalities { VOTING_BOND.with(|v| *v.borrow_mut() = self.voter_bond); TERM_DURATION.with(|v| *v.borrow_mut() = self.term_duration); DESIRED_RUNNERS_UP.with(|v| *v.borrow_mut() = self.desired_runners_up); - GenesisConfig { + MEMBERS.with(|m| *m.borrow_mut() = self.genesis_members.iter().map(|(m, _)| m.clone()).collect::>()); + let mut ext: sp_io::TestExternalities = GenesisConfig { pallet_balances: Some(pallet_balances::GenesisConfig::{ balances: vec![ (1, 10 * self.balance_factor), @@ -1040,7 +1099,12 @@ mod tests { (6, 60 * self.balance_factor) ], }), - }.build_storage().unwrap().into() + elections_phragmen: Some(elections_phragmen::GenesisConfig:: { + members: self.genesis_members + }), + }.build_storage().unwrap().into(); + ext.execute_with(|| System::set_block_number(1)); + ext } } @@ -1061,7 +1125,6 @@ mod tests { #[test] fn params_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::desired_members(), 2); assert_eq!(Elections::term_duration(), 5); assert_eq!(Elections::election_rounds(), 0); @@ -1078,6 +1141,60 @@ mod tests { }); } + #[test] + fn genesis_members_should_work() { + ExtBuilder::default().genesis_members(vec![(1, 10), (2, 20)]).build().execute_with(|| { + System::set_block_number(1); + assert_eq!(Elections::members(), vec![(1, 10), (2, 20)]); + + assert_eq!(Elections::voting(1), (10, vec![1])); + assert_eq!(Elections::voting(2), (20, vec![2])); + + // they will persist since they have self vote. + System::set_block_number(5); + assert_ok!(Elections::end_block(System::block_number())); + + assert_eq!(Elections::members_ids(), vec![1, 2]); + }) + } + + #[test] + fn genesis_members_unsorted_should_work() { + ExtBuilder::default().genesis_members(vec![(2, 20), (1, 10)]).build().execute_with(|| { + System::set_block_number(1); + assert_eq!(Elections::members(), vec![(1, 10), (2, 20)]); + + assert_eq!(Elections::voting(1), (10, vec![1])); + assert_eq!(Elections::voting(2), (20, vec![2])); + + // they will persist since they have self vote. + System::set_block_number(5); + assert_ok!(Elections::end_block(System::block_number())); + + assert_eq!(Elections::members_ids(), vec![1, 2]); + }) + } + + #[test] + #[should_panic = "Genesis member does not have enough stake"] + fn genesis_members_cannot_over_stake_0() { + // 10 cannot lock 20 as their stake and extra genesis will panic. + ExtBuilder::default().genesis_members(vec![(1, 20), (2, 20)]).build(); + } + + #[test] + #[should_panic] + fn genesis_members_cannot_over_stake_1() { + // 10 cannot reserve 20 as voting bond and extra genesis will panic. + ExtBuilder::default().voter_bond(20).genesis_members(vec![(1, 10), (2, 20)]).build(); + } + + #[test] + #[should_panic = "Duplicate member in elections phragmen genesis: 2"] + fn genesis_members_cannot_be_duplicate() { + ExtBuilder::default().genesis_members(vec![(1, 10), (2, 10), (2, 10)]).build(); + } + #[test] fn term_duration_zero_is_passive() { ExtBuilder::default() @@ -1085,7 +1202,6 @@ mod tests { .build() .execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::term_duration(), 0); assert_eq!(Elections::desired_members(), 2); assert_eq!(Elections::election_rounds(), 0); @@ -1526,10 +1642,9 @@ mod tests { assert_eq!(balances(&5), (45, 5)); assert_ok!(Elections::report_defunct_voter(Origin::signed(5), 3)); - assert_eq!( - System::events()[7].event, - Event::elections(RawEvent::VoterReported(3, 5, true)) - ); + assert!(System::events().iter().any(|event| { + event.event == Event::elections_phragmen(RawEvent::VoterReported(3, 5, true)) + })); assert_eq!(balances(&3), (28, 0)); assert_eq!(balances(&5), (47, 5)); @@ -1555,10 +1670,9 @@ mod tests { assert_eq!(balances(&5), (45, 5)); assert_ok!(Elections::report_defunct_voter(Origin::signed(5), 4)); - assert_eq!( - System::events()[7].event, - Event::elections(RawEvent::VoterReported(4, 5, false)) - ); + assert!(System::events().iter().any(|event| { + event.event == Event::elections_phragmen(RawEvent::VoterReported(4, 5, false)) + })); assert_eq!(balances(&4), (35, 5)); assert_eq!(balances(&5), (45, 3)); @@ -1966,10 +2080,9 @@ mod tests { // 5 is an outgoing loser. will also get slashed. assert_eq!(balances(&5), (45, 2)); - assert_eq!( - System::events()[6].event, - Event::elections(RawEvent::NewTerm(vec![(4, 40), (5, 50)])), - ); + assert!(System::events().iter().any(|event| { + event.event == Event::elections_phragmen(RawEvent::NewTerm(vec![(4, 40), (5, 50)])) + })); }) } diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml index a5cf36614ece8f4dff904a6e93ff76928aba3418..6043ac4681e212afb462540552af02639c8af4bd 100644 --- a/frame/elections/Cargo.toml +++ b/frame/elections/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for elections" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -34,3 +34,6 @@ std = [ "sp-runtime/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs index 6f89c4b9386b532a9ea000fb6b0dcc8b896b6d8b..a8ea0b8c2e4555d8990ba485e749f3c49b11024c 100644 --- a/frame/elections/src/lib.rs +++ b/frame/elections/src/lib.rs @@ -30,7 +30,7 @@ use sp_runtime::{ }; use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, - weights::SimpleDispatchInfo, + weights::{Weight, SimpleDispatchInfo, WeighData}, traits::{ Currency, ExistenceRequirement, Get, LockableCurrency, LockIdentifier, BalanceStatus, OnUnbalanced, ReservableCurrency, WithdrawReason, WithdrawReasons, ChangeMembers @@ -698,11 +698,12 @@ decl_module! { >::put(count); } - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { if let Err(e) = Self::end_block(n) { print("Guru meditation"); print(e); } + SimpleDispatchInfo::default().weigh_data(()) } } } diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index b82e73d512aa6eb254a1eda22ff801b7ad768a95..2898be26ca3001200d1806297b4695a655e212ea 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -208,7 +208,7 @@ impl ExtBuilder { VOTING_FEE.with(|v| *v.borrow_mut() = self.voting_fee); PRESENT_SLASH_PER_VOTER.with(|v| *v.borrow_mut() = self.bad_presentation_punishment); DECAY_RATIO.with(|v| *v.borrow_mut() = self.decay_ratio); - GenesisConfig { + let mut ext: sp_io::TestExternalities = GenesisConfig { pallet_balances: Some(pallet_balances::GenesisConfig::{ balances: vec![ (1, 10 * self.balance_factor), @@ -225,7 +225,9 @@ impl ExtBuilder { presentation_duration: 2, term_duration: 5, }), - }.build_storage().unwrap().into() + }.build_storage().unwrap().into(); + ext.execute_with(|| System::set_block_number(1)); + ext } } diff --git a/frame/elections/src/tests.rs b/frame/elections/src/tests.rs index e26f0312903fcb61c7b94fe2d93e7ac0da77e86b..64b01f12e0cfe8c3b9971bee6e3772f4a2fb2adf 100644 --- a/frame/elections/src/tests.rs +++ b/frame/elections/src/tests.rs @@ -26,7 +26,6 @@ use frame_support::{assert_ok, assert_err, assert_noop}; #[test] fn params_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::next_vote_from(1), 4); assert_eq!(Elections::next_vote_from(4), 4); assert_eq!(Elections::next_vote_from(5), 8); @@ -408,8 +407,6 @@ fn voting_locking_stake_and_reserving_bond_works() { #[test] fn voting_without_any_candidate_count_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_eq!(Elections::candidates().len(), 0); assert_noop!( @@ -422,8 +419,6 @@ fn voting_without_any_candidate_count_should_not_work() { #[test] fn voting_setting_an_approval_vote_count_more_than_candidate_count_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); assert_eq!(Elections::candidates().len(), 1); @@ -437,8 +432,6 @@ fn voting_setting_an_approval_vote_count_more_than_candidate_count_should_not_wo #[test] fn voting_resubmitting_approvals_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true], 0, 0, 40)); @@ -456,8 +449,6 @@ fn voting_resubmitting_approvals_should_work() { #[test] fn voting_retracting_voter_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); assert_eq!(Elections::candidates().len(), 1); @@ -501,7 +492,6 @@ fn voting_retracting_voter_should_work() { #[test] fn voting_invalid_retraction_index_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); @@ -514,7 +504,6 @@ fn voting_invalid_retraction_index_should_not_work() { #[test] fn voting_overflow_retraction_index_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); @@ -525,7 +514,6 @@ fn voting_overflow_retraction_index_should_not_work() { #[test] fn voting_non_voter_retraction_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); @@ -740,7 +728,6 @@ fn retracting_inactive_voter_by_nonvoter_should_not_work() { #[test] fn candidacy_simple_candidate_submission_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), Vec::::new()); assert_eq!(Elections::candidate_reg_info(1), None); assert_eq!(Elections::candidate_reg_info(2), None); @@ -768,7 +755,6 @@ fn candidacy_submission_using_free_slot_should_work() { let mut t = new_test_ext_with_candidate_holes(); t.execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), vec![0, 0, 1]); assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); @@ -784,7 +770,6 @@ fn candidacy_submission_using_alternative_free_slot_should_work() { let mut t = new_test_ext_with_candidate_holes(); t.execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), vec![0, 0, 1]); assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); @@ -800,7 +785,6 @@ fn candidacy_submission_not_using_free_slot_should_not_work() { let mut t = new_test_ext_with_candidate_holes(); t.execute_with(|| { - System::set_block_number(1); assert_noop!( Elections::submit_candidacy(Origin::signed(4), 3), Error::::InvalidCandidateSlot @@ -811,7 +795,6 @@ fn candidacy_submission_not_using_free_slot_should_not_work() { #[test] fn candidacy_bad_candidate_slot_submission_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), Vec::::new()); assert_noop!( Elections::submit_candidacy(Origin::signed(1), 1), @@ -823,7 +806,6 @@ fn candidacy_bad_candidate_slot_submission_should_not_work() { #[test] fn candidacy_non_free_candidate_slot_submission_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), Vec::::new()); assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); assert_eq!(Elections::candidates(), vec![1]); @@ -837,7 +819,6 @@ fn candidacy_non_free_candidate_slot_submission_should_not_work() { #[test] fn candidacy_dupe_candidate_submission_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), Vec::::new()); assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); assert_eq!(Elections::candidates(), vec![1]); @@ -851,7 +832,6 @@ fn candidacy_dupe_candidate_submission_should_not_work() { #[test] fn candidacy_poor_candidate_submission_should_not_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); assert_eq!(Elections::candidates(), Vec::::new()); assert_noop!( Elections::submit_candidacy(Origin::signed(7), 0), @@ -863,8 +843,6 @@ fn candidacy_poor_candidate_submission_should_not_work() { #[test] fn election_voting_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); @@ -892,8 +870,6 @@ fn election_voting_should_work() { #[test] fn election_proxy_voting_should_work() { ExtBuilder::default().build().execute_with(|| { - System::set_block_number(1); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); >::insert(11, 1); diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index a2c9a247f1c208c441328d362cd1271c4294a981..95b3c24d8796e683c30c60d6a2cb3e3a5ffd4990 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME EVM contracts pallet" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../timestamp" } @@ -42,3 +42,6 @@ std = [ "evm/std", "pallet-timestamp/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/evm/src/backend.rs b/frame/evm/src/backend.rs index e0cb816c605393b1a90235e320aacbc34bc56d09..c610f24bb1db2d763979e978dd183c5719a731b9 100644 --- a/frame/evm/src/backend.rs +++ b/frame/evm/src/backend.rs @@ -172,7 +172,7 @@ impl<'vicinity, T: Trait> ApplyBackend for Backend<'vicinity, T> { } for log in logs { - Module::::deposit_event(Event::Log(Log { + Module::::deposit_event(Event::::Log(Log { address: log.address, topics: log.topics, data: log.data, diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs index efb4c3c4f9f7823d7ccff236f5e59faa6db57d23..a50c545a461c1937d9d58eb06a86ff372558ca66 100644 --- a/frame/evm/src/lib.rs +++ b/frame/evm/src/lib.rs @@ -127,7 +127,7 @@ pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { /// Currency type for deposit and withdraw. type Currency: Currency; /// The overarching event type. - type Event: From + Into<::Event>; + type Event: From> + Into<::Event>; /// Precompiles associated with this EVM engine. type Precompiles: Precompiles; @@ -147,11 +147,17 @@ decl_storage! { decl_event! { /// EVM events - pub enum Event { + pub enum Event where + ::AccountId, + { /// Ethereum events from contracts. Log(Log), /// A contract has been created at given address. Created(H160), + /// A deposit has been made at a given address. + BalanceDeposit(AccountId, H160, U256), + /// A withdrawal has been made from a given address. + BalanceWithdraw(AccountId, H160, U256), } } @@ -202,6 +208,7 @@ decl_module! { Accounts::mutate(&address, |account| { account.balance += bvalue; }); + Module::::deposit_event(Event::::BalanceDeposit(sender, address, bvalue)); } /// Withdraw balance from EVM into currency/balances module. @@ -225,6 +232,7 @@ decl_module! { Accounts::insert(&address, account); T::Currency::resolve_creating(&sender, imbalance); + Module::::deposit_event(Event::::BalanceWithdraw(sender, address, bvalue)); } /// Issue an EVM call operation. This is similar to a message call transaction in Ethereum. @@ -289,7 +297,7 @@ decl_module! { }, )?; - Module::::deposit_event(Event::Created(create_address)); + Module::::deposit_event(Event::::Created(create_address)); Ok(()) } @@ -327,7 +335,7 @@ decl_module! { }, )?; - Module::::deposit_event(Event::Created(create_address)); + Module::::deposit_event(Event::::Created(create_address)); Ok(()) } } diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/example-offchain-worker/Cargo.toml index 425cdb05c1f746f58e5555cca59a22bd5ca36d32..6b3452d01894f30c46fcfefb8c81f7e0125ae9df 100644 --- a/frame/example-offchain-worker/Cargo.toml +++ b/frame/example-offchain-worker/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME example pallet for offchain worker" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } @@ -32,3 +32,6 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs index 5a417fa0782f3484b4c08ddb2bc1a03f7234ed6e..ac9ac2d1eea4f73a89be9d32dbddfda6c58d3ff4 100644 --- a/frame/example-offchain-worker/src/lib.rs +++ b/frame/example-offchain-worker/src/lib.rs @@ -51,9 +51,12 @@ use sp_core::crypto::KeyTypeId; use sp_runtime::{ offchain::{http, Duration, storage::StorageValueRef}, traits::Zero, - transaction_validity::{InvalidTransaction, ValidTransaction, TransactionValidity}, + transaction_validity::{ + InvalidTransaction, ValidTransaction, TransactionValidity, TransactionSource, + TransactionPriority, + }, }; -use sp_std::{vec, vec::Vec}; +use sp_std::vec::Vec; use lite_json::json::JsonValue; #[cfg(test)] @@ -104,6 +107,12 @@ pub trait Trait: frame_system::Trait { /// /// This ensures that we only accept unsigned transactions once, every `UnsignedInterval` blocks. type UnsignedInterval: Get; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; } decl_storage! { @@ -509,7 +518,10 @@ impl frame_support::unsigned::ValidateUnsigned for Module { /// By default unsigned transactions are disallowed, but implementing the validator /// here we make sure that some particular calls (the ones produced by offchain worker) /// are being whitelisted and marked as valid. - fn validate_unsigned(call: &Self::Call) -> TransactionValidity { + fn validate_unsigned( + _source: TransactionSource, + call: &Self::Call, + ) -> TransactionValidity { // Firstly let's check that we call the right function. if let Call::submit_price_unsigned(block_number, new_price) = call { // Now let's check if the transaction has any chance to succeed. @@ -532,32 +544,33 @@ impl frame_support::unsigned::ValidateUnsigned for Module { .map(|price| if &price > new_price { price - new_price } else { new_price - price }) .unwrap_or(0); - Ok(ValidTransaction { + ValidTransaction::with_tag_prefix("ExampleOffchainWorker") // We set base priority to 2**20 to make sure it's included before any other // transactions in the pool. Next we tweak the priority depending on how much // it differs from the current average. (the more it differs the more priority it // has). - priority: (1 << 20) + avg_price as u64, + .priority(T::UnsignedPriority::get().saturating_add(avg_price as _)) // This transaction does not require anything else to go before into the pool. // In theory we could require `previous_unsigned_at` transaction to go first, // but it's not necessary in our case. - requires: vec![], + //.and_requires() + // We set the `provides` tag to be the same as `next_unsigned_at`. This makes // sure only one transaction produced after `next_unsigned_at` will ever // get to the transaction pool and will end up in the block. // We can still have multiple transactions compete for the same "spot", // and the one with higher priority will replace other one in the pool. - provides: vec![codec::Encode::encode(&(KEY_TYPE.0, next_unsigned_at))], + .and_provides(next_unsigned_at) // The transaction is only valid for next 5 blocks. After that it's // going to be revalidated by the pool. - longevity: 5, + .longevity(5) // It's fine to propagate that transaction to other peers, which means it can be // created even by nodes that don't produce blocks. // Note that sometimes it's better to keep it for yourself (if you are the block // producer), since for instance in some schemes others may copy your solution and // claim a reward. - propagate: true, - }) + .propagate(true) + .build() } else { InvalidTransaction::Call.into() } diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/example-offchain-worker/src/tests.rs index f64503b0a92a84a33fb3d4949e2dd1c575882a8d..727c4942f688254365a301022be9abd711efd42d 100644 --- a/frame/example-offchain-worker/src/tests.rs +++ b/frame/example-offchain-worker/src/tests.rs @@ -94,6 +94,7 @@ impl frame_system::offchain::CreateTransaction for Test { parameter_types! { pub const GracePeriod: u64 = 5; pub const UnsignedInterval: u64 = 128; + pub const UnsignedPriority: u64 = 1 << 20; } impl Trait for Test { @@ -103,6 +104,7 @@ impl Trait for Test { type SubmitUnsignedTransaction = SubmitTransaction; type GracePeriod = GracePeriod; type UnsignedInterval = UnsignedInterval; + type UnsignedPriority = UnsignedPriority; } type Example = Module; diff --git a/frame/example/Cargo.toml b/frame/example/Cargo.toml index 344ceab2f59343b5b3a1f233c110ecc179cbacfc..4053cc0b1a51a47d17a23566afc36c1798b34988 100644 --- a/frame/example/Cargo.toml +++ b/frame/example/Cargo.toml @@ -10,8 +10,7 @@ description = "FRAME example pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking" } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } @@ -19,6 +18,8 @@ sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../. sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } + [dev-dependencies] sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core", default-features = false } @@ -35,3 +36,7 @@ std = [ "sp-io/std", "sp-std/std" ] +runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index d6036b3af85c521b366b209547b20da02592ab52..13985671c2e796f61a1c077fab94f3b12094b53e 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -256,14 +256,17 @@ use sp_std::marker::PhantomData; use frame_support::{ dispatch::DispatchResult, decl_module, decl_storage, decl_event, - weights::{SimpleDispatchInfo, DispatchInfo, DispatchClass, ClassifyDispatch, WeighData, Weight, PaysFee}, + weights::{ + SimpleDispatchInfo, DispatchClass, ClassifyDispatch, WeighData, Weight, PaysFee, + }, }; use sp_std::prelude::*; -use frame_benchmarking::{benchmarks, account}; -use frame_system::{self as system, ensure_signed, ensure_root, RawOrigin}; +use frame_system::{self as system, ensure_signed, ensure_root}; use codec::{Encode, Decode}; use sp_runtime::{ - traits::{SignedExtension, Bounded, SaturatedConversion}, + traits::{ + SignedExtension, Bounded, SaturatedConversion, DispatchInfoOf, + }, transaction_validity::{ ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity, }, @@ -516,14 +519,14 @@ decl_module! { // This function could also very well have a weight annotation, similar to any other. The // only difference being that if it is not annotated, the default is // `SimpleDispatchInfo::zero()`, which resolves into no weight. - #[weight = SimpleDispatchInfo::FixedNormal(1000)] - fn on_initialize(_n: T::BlockNumber) { + fn on_initialize(_n: T::BlockNumber) -> Weight { // Anything that needs to be done at the start of the block. // We don't do anything here. + + SimpleDispatchInfo::default().weigh_data(()) } // The signature could also look like: `fn on_finalize()` - #[weight = SimpleDispatchInfo::FixedNormal(2000)] fn on_finalize(_n: T::BlockNumber) { // Anything that needs to be done at the end of the block. // We just kill our dummy storage item. @@ -617,7 +620,6 @@ impl SignedExtension for WatchDummy { // other pallets. type Call = Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } @@ -626,7 +628,7 @@ impl SignedExtension for WatchDummy { &self, _who: &Self::AccountId, call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { // if the transaction is too big, just drop it. @@ -648,39 +650,61 @@ impl SignedExtension for WatchDummy { } } -benchmarks!{ - _ { - // Define a common range for `b`. - let b in 1 .. 1000 => (); +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking { + use super::*; + use frame_benchmarking::{benchmarks, account}; + use frame_system::RawOrigin; + + benchmarks!{ + _ { + // Define a common range for `b`. + let b in 1 .. 1000 => (); + } + + // This will measure the execution time of `accumulate_dummy` for b in [1..1000] range. + accumulate_dummy { + let b in ...; + let caller = account("caller", 0, 0); + }: _ (RawOrigin::Signed(caller), b.into()) + + // This will measure the execution time of `set_dummy` for b in [1..1000] range. + set_dummy { + let b in ...; + }: set_dummy (RawOrigin::Root, b.into()) + + // This will measure the execution time of `set_dummy` for b in [1..10] range. + another_set_dummy { + let b in 1 .. 10; + }: set_dummy (RawOrigin::Root, b.into()) + + // This will measure the execution time of sorting a vector. + sort_vector { + let x in 0 .. 10000; + let mut m = Vec::::new(); + for i in (0..x).rev() { + m.push(i); + } + }: { + m.sort(); + } } - // This will measure the execution time of `accumulate_dummy` for b in [1..1000] range. - accumulate_dummy { - let b in ...; - let caller = account("caller", 0, 0); - }: _ (RawOrigin::Signed(caller), b.into()) - - // This will measure the execution time of `set_dummy` for b in [1..1000] range. - set_dummy { - let b in ...; - let caller = account("caller", 0, 0); - }: set_dummy (RawOrigin::Signed(caller), b.into()) - - // This will measure the execution time of `set_dummy` for b in [1..10] range. - another_set_dummy { - let b in 1 .. 10; - let caller = account("caller", 0, 0); - }: set_dummy (RawOrigin::Signed(caller), b.into()) - - // This will measure the execution time of sorting a vector. - sort_vector { - let x in 0 .. 10000; - let mut m = Vec::::new(); - for i in 0..x { - m.push(i); + #[cfg(test)] + mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_accumulate_dummy::()); + assert_ok!(test_benchmark_set_dummy::()); + assert_ok!(test_benchmark_another_set_dummy::()); + assert_ok!(test_benchmark_sort_vector::()); + }); } - }: { - m.sort(); } } @@ -688,14 +712,17 @@ benchmarks!{ mod tests { use super::*; - use frame_support::{assert_ok, impl_outer_origin, parameter_types, weights::GetDispatchInfo}; + use frame_support::{ + assert_ok, impl_outer_origin, parameter_types, weights::{DispatchInfo, GetDispatchInfo}, + traits::{OnInitialize, OnFinalize} + }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ Perbill, testing::Header, - traits::{BlakeTwo256, OnInitialize, OnFinalize, IdentityLookup}, + traits::{BlakeTwo256, IdentityLookup}, }; impl_outer_origin! { @@ -752,7 +779,7 @@ mod tests { // This function basically just builds a genesis storage key/value store according to // our desired mockup. - fn new_test_ext() -> sp_io::TestExternalities { + pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_balances::GenesisConfig::::default().assimilate_storage(&mut t).unwrap(); @@ -802,13 +829,13 @@ mod tests { let info = DispatchInfo::default(); assert_eq!( - WatchDummy::(PhantomData).validate(&1, &call, info, 150) + WatchDummy::(PhantomData).validate(&1, &call, &info, 150) .unwrap() .priority, Bounded::max_value(), ); assert_eq!( - WatchDummy::(PhantomData).validate(&1, &call, info, 250), + WatchDummy::(PhantomData).validate(&1, &call, &info, 250), InvalidTransaction::ExhaustsResources.into(), ); }) diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 3ecd9753b0e7e54350a2548054e0588009272b5b..3c494199cb2374e07a45b1b888d3af52545681b8 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME executives engine" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } @@ -35,3 +35,6 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 1b3867cf6e01d887089dbfde5d3d4ec0cabb3c1b..f46fff1a3386eb4587fc0a8c104be8e684599df4 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -59,12 +59,14 @@ //! # pub type Balances = u64; //! # pub type AllModules = u64; //! # pub enum Runtime {}; -//! # use sp_runtime::transaction_validity::{TransactionValidity, UnknownTransaction}; +//! # use sp_runtime::transaction_validity::{ +//! TransactionValidity, UnknownTransaction, TransactionSource, +//! # }; //! # use sp_runtime::traits::ValidateUnsigned; //! # impl ValidateUnsigned for Runtime { //! # type Call = (); //! # -//! # fn validate_unsigned(_call: &Self::Call) -> TransactionValidity { +//! # fn validate_unsigned(_source: TransactionSource, _call: &Self::Call) -> TransactionValidity { //! # UnknownTransaction::NoUnsignedValidator.into() //! # } //! # } @@ -75,16 +77,18 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::{prelude::*, marker::PhantomData}; -use frame_support::{storage::StorageValue, weights::{GetDispatchInfo, WeighBlock, DispatchInfo}}; +use frame_support::{ + storage::StorageValue, weights::{GetDispatchInfo, DispatchInfo}, + traits::{OnInitialize, OnFinalize, OnRuntimeUpgrade, OffchainWorker}, +}; use sp_runtime::{ generic::Digest, ApplyExtrinsicResult, traits::{ - self, Header, Zero, One, Checkable, Applyable, CheckEqual, OnFinalize, OnInitialize, - NumberFor, Block as BlockT, OffchainWorker, Dispatchable, Saturating, OnRuntimeUpgrade, + self, Header, Zero, One, Checkable, Applyable, CheckEqual, ValidateUnsigned, NumberFor, + Block as BlockT, Dispatchable, Saturating, }, - transaction_validity::TransactionValidity, + transaction_validity::{TransactionValidity, TransactionSource}, }; -use sp_runtime::traits::ValidateUnsigned; use codec::{Codec, Encode}; use frame_system::{extrinsics_root, DigestOf}; @@ -111,15 +115,14 @@ impl< OnRuntimeUpgrade + OnInitialize + OnFinalize + - OffchainWorker + - WeighBlock, + OffchainWorker, > ExecuteBlock for Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: - Applyable + + Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -137,15 +140,14 @@ impl< OnRuntimeUpgrade + OnInitialize + OnFinalize + - OffchainWorker + - WeighBlock, + OffchainWorker, > Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: - Applyable + + Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -179,10 +181,8 @@ where if Self::runtime_upgraded() { // System is not part of `AllModules`, so we need to call this manually. as OnRuntimeUpgrade>::on_runtime_upgrade(); - ::on_runtime_upgrade(); - >::register_extra_weight_unchecked( - >::on_runtime_upgrade() - ); + let weight = ::on_runtime_upgrade(); + >::register_extra_weight_unchecked(weight); } >::initialize( block_number, @@ -192,13 +192,8 @@ where frame_system::InitKind::Full, ); as OnInitialize>::on_initialize(*block_number); - >::on_initialize(*block_number); - >::register_extra_weight_unchecked( - >::on_initialize(*block_number) - ); - >::register_extra_weight_unchecked( - >::on_finalize(*block_number) - ); + let weight = >::on_initialize(*block_number); + >::register_extra_weight_unchecked(weight); frame_system::Module::::note_finished_initialize(); } @@ -312,7 +307,7 @@ where // Decode parameters and dispatch let dispatch_info = xt.get_dispatch_info(); - let r = Applyable::apply::(xt, dispatch_info, encoded_len)?; + let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; >::note_applied_extrinsic(&r, encoded_len as u32, dispatch_info); @@ -345,12 +340,15 @@ where /// side-effects; it merely checks whether the transaction would panic if it were included or not. /// /// Changes made to storage should be discarded. - pub fn validate_transaction(uxt: Block::Extrinsic) -> TransactionValidity { + pub fn validate_transaction( + source: TransactionSource, + uxt: Block::Extrinsic, + ) -> TransactionValidity { let encoded_len = uxt.using_encoded(|d| d.len()); let xt = uxt.check(&Default::default())?; let dispatch_info = xt.get_dispatch_info(); - xt.validate::(dispatch_info, encoded_len) + xt.validate::(source, &dispatch_info, encoded_len) } /// Start an offchain worker and generate extrinsics. @@ -400,7 +398,7 @@ mod tests { use hex_literal::hex; mod custom { - use frame_support::weights::SimpleDispatchInfo; + use frame_support::weights::{SimpleDispatchInfo, Weight}; pub trait Trait: frame_system::Trait {} @@ -422,11 +420,11 @@ mod tests { // module hooks. // one with block number arg and one without - #[weight = SimpleDispatchInfo::FixedNormal(25)] - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { println!("on_initialize({})", n); + 175 } - #[weight = SimpleDispatchInfo::FixedNormal(150)] + fn on_finalize() { println!("on_finalize(?)"); } @@ -518,7 +516,10 @@ mod tests { Ok(()) } - fn validate_unsigned(call: &Self::Call) -> TransactionValidity { + fn validate_unsigned( + _source: TransactionSource, + call: &Self::Call, + ) -> TransactionValidity { match call { Call::Balances(BalancesCall::set_balance(_, _, _)) => Ok(Default::default()), _ => UnknownTransaction::NoUnsignedValidator.into(), @@ -732,7 +733,10 @@ mod tests { let mut t = new_test_ext(1); t.execute_with(|| { - assert_eq!(Executive::validate_transaction(xt.clone()), Ok(Default::default())); + assert_eq!( + Executive::validate_transaction(TransactionSource::InBlock, xt.clone()), + Ok(Default::default()), + ); assert_eq!(Executive::apply_extrinsic(xt), Ok(Err(DispatchError::BadOrigin))); }); } diff --git a/frame/finality-tracker/Cargo.toml b/frame/finality-tracker/Cargo.toml index 3c2d560e1adec19f37678bf51cc878161223cddd..e261fae05f5afd4bda7e69a0fc26e3d052890488 100644 --- a/frame/finality-tracker/Cargo.toml +++ b/frame/finality-tracker/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/pallet-finality-tracker" [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -37,3 +37,6 @@ std = [ "sp-finality-tracker/std", "sp-inherents/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/finality-tracker/src/lib.rs b/frame/finality-tracker/src/lib.rs index c7d5ad30d9a8838763f30316830eb619c03d6489..8200543ffa17136173346a5bf0fb6f9ece5277e2 100644 --- a/frame/finality-tracker/src/lib.rs +++ b/frame/finality-tracker/src/lib.rs @@ -76,6 +76,7 @@ decl_module! { /// Hint that the author of this block thinks the best finalized /// block is the given number. + #[weight = frame_support::weights::SimpleDispatchInfo::FixedMandatory(10_000)] fn final_hint(origin, #[compact] hint: T::BlockNumber) { ensure_none(origin)?; ensure!(!::Update::exists(), Error::::AlreadyUpdated); @@ -207,9 +208,11 @@ mod tests { use sp_core::H256; use sp_runtime::{ testing::Header, Perbill, - traits::{BlakeTwo256, IdentityLookup, OnFinalize, Header as HeaderT}, + traits::{BlakeTwo256, IdentityLookup, Header as HeaderT}, + }; + use frame_support::{ + assert_ok, impl_outer_origin, parameter_types, weights::Weight, traits::OnFinalize }; - use frame_support::{assert_ok, impl_outer_origin, parameter_types, weights::Weight}; use frame_system as system; use std::cell::RefCell; diff --git a/frame/generic-asset/Cargo.toml b/frame/generic-asset/Cargo.toml index 2195bc064b2948e8b809a47e94e21c074abb4133..b531a0ed9af24be72d5146a0d06ada267521a8e8 100644 --- a/frame/generic-asset/Cargo.toml +++ b/frame/generic-asset/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for generic asset management" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } @@ -30,3 +30,6 @@ std =[ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs index 3cb48ad18ae36d9c1a546edbb2037b18314d6d22..b16666cb6b7e1c5dd96616a38035d141a70f371d 100644 --- a/frame/generic-asset/src/lib.rs +++ b/frame/generic-asset/src/lib.rs @@ -360,12 +360,14 @@ decl_module! { fn deposit_event() = default; /// Create a new kind of asset. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn create(origin, options: AssetOptions) -> DispatchResult { let origin = ensure_signed(origin)?; Self::create_asset(None, Some(origin), options) } /// Transfer some liquid free balance to another account. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn transfer(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, #[compact] amount: T::Balance) { let origin = ensure_signed(origin)?; ensure!(!amount.is_zero(), Error::::ZeroAmount); @@ -375,6 +377,7 @@ decl_module! { /// Updates permission for a given `asset_id` and an account. /// /// The `origin` must have `update` permission. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn update_permission( origin, #[compact] asset_id: T::AssetId, @@ -397,6 +400,7 @@ decl_module! { /// Mints an asset, increases its total issuance. /// The origin must have `mint` permissions. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn mint(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult { let who = ensure_signed(origin)?; Self::mint_free(&asset_id, &who, &to, &amount)?; @@ -406,6 +410,7 @@ decl_module! { /// Burns an asset, decreases its total issuance. /// The `origin` must have `burn` permissions. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn burn(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult { let who = ensure_signed(origin)?; Self::burn_free(&asset_id, &who, &to, &amount)?; @@ -415,6 +420,7 @@ decl_module! { /// Can be used to create reserved tokens. /// Requires Root call. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn create_reserved( origin, asset_id: T::AssetId, diff --git a/frame/generic-asset/src/mock.rs b/frame/generic-asset/src/mock.rs index 8db140d90c666359cf21092a3fc4c9f49217d1e9..c805b793bc7b4e358e8c8f2b4b5045a8c14dd360 100644 --- a/frame/generic-asset/src/mock.rs +++ b/frame/generic-asset/src/mock.rs @@ -127,16 +127,17 @@ impl ExtBuilder { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); GenesisConfig:: { - assets: vec![self.asset_id], - endowed_accounts: self.accounts, - initial_balance: self.initial_balance, - next_asset_id: self.next_asset_id, - staking_asset_id: 16000, - spending_asset_id: 16001, - } - .assimilate_storage(&mut t).unwrap(); - - t.into() + assets: vec![self.asset_id], + endowed_accounts: self.accounts, + initial_balance: self.initial_balance, + next_asset_id: self.next_asset_id, + staking_asset_id: 16000, + spending_asset_id: 16001, + }.assimilate_storage(&mut t).unwrap(); + + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } } diff --git a/frame/grandpa/Cargo.toml b/frame/grandpa/Cargo.toml index 43aa5d0c6836f23c58fd22fe3fa15a0bcdf29dbe..206b563bd9555c9f163d82473788a8baf3ecc275 100644 --- a/frame/grandpa/Cargo.toml +++ b/frame/grandpa/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for GRANDPA finality gadget" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-finality-grandpa = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/finality-grandpa" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -39,3 +39,6 @@ std = [ "pallet-session/std", "pallet-finality-tracker/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs index dd0101e5f66e1cba6687ebdd3d28455742ff7270..030699b52587262eef974a923fd4074a7fb043e3 100644 --- a/frame/grandpa/src/lib.rs +++ b/frame/grandpa/src/lib.rs @@ -34,7 +34,7 @@ use sp_std::prelude::*; use codec::{self as codec, Encode, Decode}; use frame_support::{decl_event, decl_storage, decl_module, decl_error, storage}; use sp_runtime::{ - DispatchResult, generic::{DigestItem, OpaqueDigestItemId}, traits::Zero, Perbill, PerThing, + DispatchResult, generic::{DigestItem, OpaqueDigestItemId}, traits::Zero, Perbill, }; use sp_staking::{ SessionIndex, @@ -184,6 +184,7 @@ decl_module! { fn deposit_event() = default; /// Report some misbehavior. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn report_misbehavior(origin, _report: Vec) { ensure_signed(origin)?; // FIXME: https://github.com/paritytech/substrate/issues/1112 diff --git a/frame/grandpa/src/tests.rs b/frame/grandpa/src/tests.rs index 19e42a301f2f655a0fb5681ba55cf1d61b4b1db8..b583c31968d896e36e11811ff0193bea7a629ff9 100644 --- a/frame/grandpa/src/tests.rs +++ b/frame/grandpa/src/tests.rs @@ -18,7 +18,8 @@ #![cfg(test)] -use sp_runtime::{testing::{H256, Digest}, traits::{Header, OnFinalize}}; +use sp_runtime::{testing::{H256, Digest}, traits::Header}; +use frame_support::traits::OnFinalize; use crate::mock::*; use frame_system::{EventRecord, Phase}; use codec::{Decode, Encode}; diff --git a/frame/identity/Cargo.toml b/frame/identity/Cargo.toml index f13970e6b2de0eb8123733ca0c71405de98ad37d..22b385d06d76f36677a50b2afcbe51ad564e970f 100644 --- a/frame/identity/Cargo.toml +++ b/frame/identity/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME identity management pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -36,3 +36,6 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/identity/src/benchmarking.rs b/frame/identity/src/benchmarking.rs index 1ca10707c9a151df85d1a8a01cf09bbef53edef4..fe99cd990720cf0daeb7b5c98fb4c2f0e6556c43 100644 --- a/frame/identity/src/benchmarking.rs +++ b/frame/identity/src/benchmarking.rs @@ -16,6 +16,8 @@ //! Identity pallet benchmarking. +#![cfg(feature = "runtime-benchmarks")] + use super::*; use frame_system::RawOrigin; @@ -278,3 +280,27 @@ benchmarks! { } }: _(RawOrigin::Root, caller_lookup) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_add_registrar::()); + assert_ok!(test_benchmark_set_identity::()); + assert_ok!(test_benchmark_set_subs::()); + assert_ok!(test_benchmark_clear_identity::()); + assert_ok!(test_benchmark_request_judgement::()); + assert_ok!(test_benchmark_cancel_request::()); + assert_ok!(test_benchmark_set_fee::()); + assert_ok!(test_benchmark_set_account_id::()); + assert_ok!(test_benchmark_set_fields::()); + assert_ok!(test_benchmark_provide_judgement::()); + assert_ok!(test_benchmark_kill_identity::()); + }); + } +} diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs index 4cbd8c7a2b574a900ba14fe6627e55df18d2b678..2a2d1c9cf8404d68a956d1d9fe098bfc2cc8f042 100644 --- a/frame/identity/src/lib.rs +++ b/frame/identity/src/lib.rs @@ -70,16 +70,15 @@ use sp_std::{fmt::Debug, ops::Add, iter::once}; use enumflags2::BitFlags; use codec::{Encode, Decode}; use sp_runtime::{DispatchResult, RuntimeDebug}; -use sp_runtime::traits::{StaticLookup, EnsureOrigin, Zero, AppendZerosInput}; +use sp_runtime::traits::{StaticLookup, Zero, AppendZerosInput}; use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, - traits::{Currency, ReservableCurrency, OnUnbalanced, Get, BalanceStatus}, + traits::{Currency, ReservableCurrency, OnUnbalanced, Get, BalanceStatus, EnsureOrigin}, weights::SimpleDispatchInfo, }; use frame_system::{self as system, ensure_signed, ensure_root}; -#[cfg(feature = "runtime-benchmarks")] -pub mod benchmarking; +mod benchmarking; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; @@ -978,7 +977,7 @@ mod tests { // This function basically just builds a genesis storage key/value store according to // our desired mockup. - fn new_test_ext() -> sp_io::TestExternalities { + pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_balances::GenesisConfig:: { diff --git a/frame/im-online/Cargo.toml b/frame/im-online/Cargo.toml index 5d5d40079eb7a11522767fe5946be4547b79a384..dabcc45ef166af7fdf5c47e6a86bbc41732017a0 100644 --- a/frame/im-online/Cargo.toml +++ b/frame/im-online/Cargo.toml @@ -11,7 +11,7 @@ description = "FRAME's I'm online pallet" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../authorship" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } @@ -41,3 +41,6 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs index 1269328634894a3d8ed3e44440d26a4b67d7bd98..e0e74bccfa35da9a7954daa266014ae671fc5a18 100644 --- a/frame/im-online/src/benchmarking.rs +++ b/frame/im-online/src/benchmarking.rs @@ -24,6 +24,7 @@ use frame_system::RawOrigin; use frame_benchmarking::benchmarks; use sp_core::offchain::{OpaquePeerId, OpaqueMultiaddr}; use sp_runtime::traits::{ValidateUnsigned, Zero}; +use sp_runtime::transaction_validity::TransactionSource; use crate::Module as ImOnline; @@ -72,52 +73,21 @@ benchmarks! { let (input_heartbeat, signature) = create_heartbeat::(k, e)?; let call = Call::heartbeat(input_heartbeat, signature); }: { - ImOnline::::validate_unsigned(&call)?; + ImOnline::::validate_unsigned(TransactionSource::InBlock, &call)?; } } #[cfg(test)] mod tests { - use crate::*; - use super::SelectedBenchmark; - use crate::mock::*; + use super::*; + use crate::mock::{new_test_ext, Runtime}; use frame_support::assert_ok; #[test] - fn test_heartbeat_benchmark() { + fn test_benchmarks() { new_test_ext().execute_with(|| { - let k = 10; - - assert_eq!(ReceivedHeartbeats::iter_prefix(0).count(), 0); - - let selected_benchmark = SelectedBenchmark::heartbeat; - let c = vec![(frame_benchmarking::BenchmarkParameter::k, k)]; - let closure_to_benchmark = - >::instance( - &selected_benchmark, - &c - ).unwrap(); - - assert_ok!(closure_to_benchmark()); - - assert_eq!(ReceivedHeartbeats::iter_prefix(0).count(), 1); - }); - } - - #[test] - fn test_validate_unsigned_benchmark() { - new_test_ext().execute_with(|| { - let k = 10; - - let selected_benchmark = SelectedBenchmark::validate_unsigned; - let c = vec![(frame_benchmarking::BenchmarkParameter::k, k)]; - let closure_to_benchmark = - >::instance( - &selected_benchmark, - &c - ).unwrap(); - - assert_ok!(closure_to_benchmark()); + assert_ok!(test_benchmark_heartbeat::()); + assert_ok!(test_benchmark_validate_unsigned::()); }); } } diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs index 861c57e5b61bceba289b3a9331d624d75febf3b9..9c2b55a5c037e4833722253b0ee9bab775d5ce86 100644 --- a/frame/im-online/src/lib.rs +++ b/frame/im-online/src/lib.rs @@ -50,6 +50,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] //! pub fn is_online(origin, authority_index: u32) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _is_online = >::is_online(authority_index); @@ -80,9 +81,9 @@ use pallet_session::historical::IdentificationTuple; use sp_runtime::{ offchain::storage::StorageValueRef, RuntimeDebug, - traits::{Convert, Member, Saturating, AtLeast32Bit}, Perbill, PerThing, + traits::{Convert, Member, Saturating, AtLeast32Bit}, Perbill, transaction_validity::{ - TransactionValidity, ValidTransaction, InvalidTransaction, + TransactionValidity, ValidTransaction, InvalidTransaction, TransactionSource, TransactionPriority, }, }; @@ -246,6 +247,12 @@ pub trait Trait: frame_system::Trait + pallet_session::historical::Trait { IdentificationTuple, UnresponsivenessOffence>, >; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; } decl_event!( @@ -309,6 +316,7 @@ decl_module! { fn deposit_event() = default; + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn heartbeat( origin, heartbeat: Heartbeat, @@ -624,7 +632,10 @@ impl pallet_session::OneSessionHandler for Module { impl frame_support::unsigned::ValidateUnsigned for Module { type Call = Call; - fn validate_unsigned(call: &Self::Call) -> TransactionValidity { + fn validate_unsigned( + _source: TransactionSource, + call: &Self::Call, + ) -> TransactionValidity { if let Call::heartbeat(heartbeat, signature) = call { if >::is_online(heartbeat.authority_index) { // we already received a heartbeat for this authority @@ -653,13 +664,14 @@ impl frame_support::unsigned::ValidateUnsigned for Module { return InvalidTransaction::BadProof.into(); } - Ok(ValidTransaction { - priority: TransactionPriority::max_value(), - requires: vec![], - provides: vec![(current_session, authority_id).encode()], - longevity: TryInto::::try_into(T::SessionDuration::get() / 2.into()).unwrap_or(64_u64), - propagate: true, - }) + ValidTransaction::with_tag_prefix("ImOnline") + .priority(T::UnsignedPriority::get()) + .and_provides((current_session, authority_id)) + .longevity(TryInto::::try_into( + T::SessionDuration::get() / 2.into() + ).unwrap_or(64_u64)) + .propagate(true) + .build() } else { InvalidTransaction::Call.into() } @@ -674,11 +686,11 @@ pub struct UnresponsivenessOffence { /// /// It acts as a time measure for unresponsiveness reports and effectively will always point /// at the end of the session. - session_index: SessionIndex, + pub session_index: SessionIndex, /// The size of the validator set in current session/era. - validator_set_count: u32, + pub validator_set_count: u32, /// Authorities that were unresponsive during the current era. - offenders: Vec, + pub offenders: Vec, } impl Offence for UnresponsivenessOffence { diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index 78b6409d543eb366ca77e5b94a1cea2ac92973cb..73ccaf3f707cbc7f37f7902dd1ba11d7e4e73296 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -141,6 +141,7 @@ impl pallet_session::Trait for Runtime { type Keys = UintAuthorityId; type Event = (); type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = pallet_session::PeriodicSessions; } impl pallet_session::historical::Trait for Runtime { @@ -159,6 +160,10 @@ impl pallet_authorship::Trait for Runtime { type EventHandler = ImOnline; } +parameter_types! { + pub const UnsignedPriority: u64 = 1 << 20; +} + impl Trait for Runtime { type AuthorityId = UintAuthorityId; type Event = (); @@ -166,6 +171,7 @@ impl Trait for Runtime { type SubmitTransaction = SubmitTransaction; type ReportUnresponsiveness = OffenceHandler; type SessionDuration = Period; + type UnsignedPriority = UnsignedPriority; } /// Im Online module. @@ -174,7 +180,7 @@ pub type System = frame_system::Module; pub type Session = pallet_session::Module; pub fn advance_session() { - let now = System::block_number(); + let now = System::block_number().max(1); System::set_block_number(now + 1); Session::rotate_session(); assert_eq!(Session::current_index(), (now / Period::get()) as u32); diff --git a/frame/im-online/src/tests.rs b/frame/im-online/src/tests.rs index b43adca0fd485d36b5c8d2f8d81ce365d8518711..c7bf2afcca629f74765c1519abdd063e63dae1f1 100644 --- a/frame/im-online/src/tests.rs +++ b/frame/im-online/src/tests.rs @@ -190,7 +190,7 @@ fn late_heartbeat_should_fail() { #[test] fn should_generate_heartbeats() { - use sp_runtime::traits::OffchainWorker; + use frame_support::traits::OffchainWorker; let mut ext = new_test_ext(); let (offchain, _state) = TestOffchainExt::new(); diff --git a/frame/indices/Cargo.toml b/frame/indices/Cargo.toml index 859461bfe79e009fe6327f9182124405ceb9c7af..f28f393642bedac28e1644f35046d6ad9c1b4a2e 100644 --- a/frame/indices/Cargo.toml +++ b/frame/indices/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME indices management pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -35,3 +35,6 @@ std = [ "sp-runtime/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs index 95ac6cf75283816bb3058f148d849f365de1345d..d2ba664d425ffce1ef0aac0745fea825d7183a42 100644 --- a/frame/indices/src/lib.rs +++ b/frame/indices/src/lib.rs @@ -25,6 +25,7 @@ use sp_runtime::traits::{ StaticLookup, Member, LookupError, Zero, One, BlakeTwo256, Hash, Saturating, AtLeast32Bit }; use frame_support::{Parameter, decl_module, decl_error, decl_event, decl_storage, ensure}; +use frame_support::weights::{Weight, SimpleDispatchInfo, WeighData}; use frame_support::dispatch::DispatchResult; use frame_support::traits::{Currency, ReservableCurrency, Get, BalanceStatus::Reserved}; use frame_support::storage::migration::take_storage_value; @@ -98,8 +99,10 @@ decl_module! { pub struct Module for enum Call where origin: T::Origin, system = frame_system { fn deposit_event() = default; - fn on_initialize() { + fn on_initialize() -> Weight { Self::migrations(); + + SimpleDispatchInfo::default().weigh_data(()) } /// Assign an previously unassigned index. @@ -118,6 +121,7 @@ decl_module! { /// - One reserve operation. /// - One event. /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn claim(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; @@ -145,6 +149,7 @@ decl_module! { /// - One transfer operation. /// - One event. /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn transfer(origin, new: T::AccountId, index: T::AccountIndex) { let who = ensure_signed(origin)?; ensure!(who != new, Error::::NotTransfer); @@ -175,6 +180,7 @@ decl_module! { /// - One reserve operation. /// - One event. /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn free(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; @@ -203,6 +209,7 @@ decl_module! { /// - Up to one reserve operation. /// - One event. /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn force_transfer(origin, new: T::AccountId, index: T::AccountIndex) { ensure_root(origin)?; diff --git a/frame/membership/Cargo.toml b/frame/membership/Cargo.toml index 9272d1c76bba81c3486a1273adb3c2a0e7056c1a..41e56b584f550332fe1b91f5763fa31765086d75 100644 --- a/frame/membership/Cargo.toml +++ b/frame/membership/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME membership management pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } @@ -31,3 +31,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 129f3c4003bdd306a8feedcd4f2d39ffb6ddfa72..8f086fa2f32114a6b2e17bc8d5f228cfee070d08 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -25,11 +25,10 @@ use sp_std::prelude::*; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, - traits::{ChangeMembers, InitializeMembers}, + traits::{ChangeMembers, InitializeMembers, EnsureOrigin}, weights::SimpleDispatchInfo, }; use frame_system::{self as system, ensure_root, ensure_signed}; -use sp_runtime::traits::EnsureOrigin; pub trait Trait: frame_system::Trait { /// The overarching event type. diff --git a/frame/metadata/Cargo.toml b/frame/metadata/Cargo.toml index 1719d98f345d9f5cb4a317502232a1063e48617b..f965e1dddcb09687f4cdc193c62f09e60f4d12e5 100644 --- a/frame/metadata/Cargo.toml +++ b/frame/metadata/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Decodable variant of the RuntimeMetadata." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } @@ -22,3 +22,6 @@ std = [ "sp-core/std", "serde", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/nicks/Cargo.toml b/frame/nicks/Cargo.toml index 5c21fde7052f9a81aecec1207d20e99206f70045..ea88021d2528b8897c02a721ffc6fcf64564358d 100644 --- a/frame/nicks/Cargo.toml +++ b/frame/nicks/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for nick management" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -32,3 +32,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index 2ab1789d395d367ce5e44752aa55540b36d89061..ae005e2500b85ab3da72aa94746681d1eaa01f3a 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -40,11 +40,11 @@ use sp_std::prelude::*; use sp_runtime::{ - traits::{StaticLookup, EnsureOrigin, Zero} + traits::{StaticLookup, Zero} }; use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, - traits::{Currency, ReservableCurrency, OnUnbalanced, Get}, + traits::{Currency, EnsureOrigin, ReservableCurrency, OnUnbalanced, Get}, weights::SimpleDispatchInfo, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -171,6 +171,7 @@ decl_module! { /// - One storage read/write. /// - One event. /// # + #[weight = SimpleDispatchInfo::FixedNormal(70_000)] fn clear_name(origin) { let sender = ensure_signed(origin)?; diff --git a/frame/offences/Cargo.toml b/frame/offences/Cargo.toml index 31f7b1d8f44e56d0a217ea620b0433115c57b7dd..eab95dbd048e16054065aba215e726144836cdd7 100644 --- a/frame/offences/Cargo.toml +++ b/frame/offences/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME offences pallet" [dependencies] pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -34,3 +34,7 @@ std = [ "frame-support/std", "frame-system/std", ] +runtime-benchmarks = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/offences/benchmarking/Cargo.toml b/frame/offences/benchmarking/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e343f1ff0c1ee18cf8d7bb6d73e8c5b535d80d35 --- /dev/null +++ b/frame/offences/benchmarking/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "pallet-offences-benchmarking" +version = "2.0.0-alpha.5" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME offences pallet benchmarking" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } + +sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } +sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/staking" } +sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../../benchmarking" } +frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../../system" } +frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../support" } +pallet-im-online = { version = "2.0.0-alpha.5", default-features = false, path = "../../im-online" } +pallet-offences = { version = "2.0.0-alpha.5", default-features = false, features = ["runtime-benchmarks"], path = "../../offences" } +pallet-staking = { version = "2.0.0-alpha.5", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } +pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../session" } +sp-io = { path = "../../../primitives/io", default-features = false, version = "2.0.0-alpha.5"} + + +[features] +default = ["std"] +std = [ + "sp-runtime/std", + "sp-std/std", + "sp-staking/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "pallet-offences/std", + "pallet-im-online/std", + "pallet-staking/std", + "pallet-session/std", +] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/offences/benchmarking/src/lib.rs b/frame/offences/benchmarking/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a88714a89a7faf82176d0e1feed568db42f8b285 --- /dev/null +++ b/frame/offences/benchmarking/src/lib.rs @@ -0,0 +1,175 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Offences pallet benchmarking. + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use sp_std::vec; + +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use frame_support::traits::{Currency, OnInitialize}; + +use sp_runtime::{Perbill, traits::{Convert, StaticLookup}}; +use sp_staking::offence::ReportOffence; + +use pallet_im_online::{Trait as ImOnlineTrait, Module as ImOnline, UnresponsivenessOffence}; +use pallet_offences::{Trait as OffencesTrait, Module as Offences}; +use pallet_staking::{ + Module as Staking, Trait as StakingTrait, RewardDestination, ValidatorPrefs, + Exposure, IndividualExposure, ElectionStatus +}; +use pallet_session::Trait as SessionTrait; +use pallet_session::historical::{Trait as HistoricalTrait, IdentificationTuple}; + +const SEED: u32 = 0; + +const MAX_USERS: u32 = 1000; +const MAX_REPORTERS: u32 = 100; +const MAX_OFFENDERS: u32 = 100; +const MAX_NOMINATORS: u32 = 100; +const MAX_DEFERRED_OFFENCES: u32 = 100; + +pub struct Module(Offences); + +pub trait Trait: SessionTrait + StakingTrait + OffencesTrait + ImOnlineTrait + HistoricalTrait {} + +fn create_offender(n: u32, nominators: u32) -> Result { + let stash: T::AccountId = account("stash", n, SEED); + let controller: T::AccountId = account("controller", n, SEED); + let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let reward_destination = RewardDestination::Staked; + let amount = T::Currency::minimum_balance(); + + Staking::::bond( + RawOrigin::Signed(stash.clone()).into(), + controller_lookup.clone(), + amount.clone(), + reward_destination.clone(), + )?; + + let validator_prefs = ValidatorPrefs { + commission: Perbill::from_percent(50), + }; + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), validator_prefs)?; + + let mut individual_exposures = vec![]; + + // Create n nominators + for i in 0 .. nominators { + let nominator_stash: T::AccountId = account("nominator stash", n * MAX_NOMINATORS + i, SEED); + let nominator_controller: T::AccountId = account("nominator controller", n * MAX_NOMINATORS + i, SEED); + let nominator_controller_lookup: ::Source = T::Lookup::unlookup(nominator_controller.clone()); + + Staking::::bond( + RawOrigin::Signed(nominator_stash.clone()).into(), + nominator_controller_lookup.clone(), + amount, + reward_destination, + )?; + + let selected_validators: Vec<::Source> = vec![controller_lookup.clone()]; + Staking::::nominate(RawOrigin::Signed(nominator_controller.clone()).into(), selected_validators)?; + + individual_exposures.push(IndividualExposure { + who: nominator_controller.clone(), + value: amount.clone(), + }); + } + + let exposure = Exposure { + total: amount.clone() * n.into(), + own: amount, + others: individual_exposures, + }; + let current_era = 0u32; + Staking::::add_era_stakers(current_era.into(), stash.clone().into(), exposure); + + Ok(controller) +} + +fn make_offenders(num_offenders: u32, num_nominators: u32) -> Result>, &'static str> { + let mut offenders: Vec = vec![]; + + for i in 0 .. num_offenders { + let offender = create_offender::(i, num_nominators)?; + offenders.push(offender); + } + + Ok(offenders.iter() + .map(|id| + ::ValidatorIdOf::convert(id.clone()) + .expect("failed to get validator id from account id")) + .map(|validator_id| + ::FullIdentificationOf::convert(validator_id.clone()) + .map(|full_id| (validator_id, full_id)) + .expect("failed to convert validator id to full identification")) + .collect::>>()) +} + +benchmarks! { + _ { + let u in 1 .. MAX_USERS => (); + let r in 1 .. MAX_REPORTERS => (); + let o in 1 .. MAX_OFFENDERS => (); + let n in 1 .. MAX_NOMINATORS => (); + let d in 1 .. MAX_DEFERRED_OFFENCES => (); + } + + report_offence { + let r in ...; + let o in ...; + let n in ...; + + let mut reporters = vec![]; + + for i in 0 .. r { + let reporter = account("reporter", i, SEED); + reporters.push(reporter); + } + + let offenders = make_offenders::(o, n).expect("failed to create offenders"); + let keys = ImOnline::::keys(); + + let offence = UnresponsivenessOffence { + session_index: 0, + validator_set_count: keys.len() as u32, + offenders, + }; + + }: { + let _ = ::ReportUnresponsiveness::report_offence(reporters, offence); + } + + on_initialize { + let d in ...; + + Staking::::put_election_status(ElectionStatus::Closed); + + let mut deferred_offences = vec![]; + + for i in 0 .. d { + deferred_offences.push((vec![], vec![], 0u32)); + } + + Offences::::set_deferred_offences(deferred_offences); + + }: { + Offences::::on_initialize(u.into()); + } +} diff --git a/frame/offences/src/lib.rs b/frame/offences/src/lib.rs index 325ceceb8b8855f4671cdbf145ed9b825c25d0b1..40f39ab5f2a0b3519a9878eae5e9b1d5568c2d7e 100644 --- a/frame/offences/src/lib.rs +++ b/frame/offences/src/lib.rs @@ -26,10 +26,12 @@ mod tests; use sp_std::vec::Vec; use frame_support::{ - decl_module, decl_event, decl_storage, Parameter, + decl_module, decl_event, decl_storage, Parameter, debug, + weights::{Weight, SimpleDispatchInfo, WeighData}, }; -use sp_runtime::traits::Hash; +use sp_runtime::{traits::Hash, Perbill}; use sp_staking::{ + SessionIndex, offence::{Offence, ReportOffence, Kind, OnOffenceHandler, OffenceDetails, OffenceError}, }; use codec::{Encode, Decode}; @@ -41,6 +43,13 @@ type OpaqueTimeSlot = Vec; /// A type alias for a report identifier. type ReportIdOf = ::Hash; +/// Type of data stored as a deferred offence +pub type DeferredOffenceOf = ( + Vec::AccountId, ::IdentificationTuple>>, + Vec, + SessionIndex, +); + /// Offences trait pub trait Trait: frame_system::Trait { /// The overarching event type. @@ -58,6 +67,10 @@ decl_storage! { map hasher(twox_64_concat) ReportIdOf => Option>; + /// Deferred reports that have been rejected by the offence handler and need to be submitted + /// at a later time. + DeferredOffences get(deferred_offences): Vec>; + /// A vector of reports of the same kind that happened at the same time slot. ConcurrentReportsIndex: double_map hasher(twox_64_concat) Kind, hasher(twox_64_concat) OpaqueTimeSlot @@ -76,20 +89,43 @@ decl_storage! { decl_event!( pub enum Event { /// There is an offence reported of the given `kind` happened at the `session_index` and - /// (kind-specific) time slot. This event is not deposited for duplicate slashes. - Offence(Kind, OpaqueTimeSlot), + /// (kind-specific) time slot. This event is not deposited for duplicate slashes. last + /// element indicates of the offence was applied (true) or queued (false). + Offence(Kind, OpaqueTimeSlot, bool), } ); decl_module! { - /// Offences module, currently just responsible for taking offence reports. pub struct Module for enum Call where origin: T::Origin { fn deposit_event() = default; - fn on_runtime_upgrade() { + fn on_runtime_upgrade() -> Weight { Reports::::remove_all(); ConcurrentReportsIndex::::remove_all(); ReportsByKindIndex::remove_all(); + + SimpleDispatchInfo::default().weigh_data(()) + } + + fn on_initialize(now: T::BlockNumber) -> Weight { + // only decode storage if we can actually submit anything again. + if T::OnOffenceHandler::can_report() { + >::mutate(|deferred| { + // keep those that fail to be reported again. An error log is emitted here; this + // should not happen if staking's `can_report` is implemented properly. + deferred.retain(|(o, p, s)| { + T::OnOffenceHandler::on_offence(&o, &p, *s).map_err(|_| { + debug::native::error!( + target: "pallet-offences", + "re-submitting a deferred slash returned Err at {}. This should not happen with pallet-staking", + now, + ); + }).is_err() + }) + }) + } + + SimpleDispatchInfo::default().weigh_data(()) } } } @@ -116,9 +152,6 @@ where None => return Err(OffenceError::DuplicateReport), }; - // Deposit the event. - Self::deposit_event(Event::Offence(O::ID, time_slot.encode())); - let offenders_count = concurrent_offenders.len() as u32; // The amount new offenders are slashed @@ -127,17 +160,42 @@ where let slash_perbill: Vec<_> = (0..concurrent_offenders.len()) .map(|_| new_fraction.clone()).collect(); - T::OnOffenceHandler::on_offence( + let applied = Self::report_or_store_offence( &concurrent_offenders, &slash_perbill, offence.session_index(), ); + // Deposit the event. + Self::deposit_event(Event::Offence(O::ID, time_slot.encode(), applied)); + Ok(()) } } impl Module { + /// Tries (without checking) to report an offence. Stores them in [`DeferredOffences`] in case + /// it fails. Returns false in case it has to store the offence. + fn report_or_store_offence( + concurrent_offenders: &[OffenceDetails], + slash_perbill: &[Perbill], + session_index: SessionIndex, + ) -> bool { + match T::OnOffenceHandler::on_offence( + &concurrent_offenders, + &slash_perbill, + session_index, + ) { + Ok(_) => true, + Err(_) => { + >::mutate(|d| + d.push((concurrent_offenders.to_vec(), slash_perbill.to_vec(), session_index)) + ); + false + } + } + } + /// Compute the ID for the given report properties. /// /// The report id depends on the offence kind, time slot and the id of offender. @@ -191,6 +249,11 @@ impl Module { None } } + + #[cfg(feature = "runtime-benchmarks")] + pub fn set_deferred_offences(offences: Vec>) { + >::put(offences); + } } struct TriageOutcome { diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index a003ad69157fcf5e8bf2f9c296aea3dd010e1b7d..e464200396ebde9be263ee75ea610813203d34c1 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -43,6 +43,7 @@ pub struct OnOffenceHandler; thread_local! { pub static ON_OFFENCE_PERBILL: RefCell> = RefCell::new(Default::default()); + pub static CAN_REPORT: RefCell = RefCell::new(true); } impl offence::OnOffenceHandler for OnOffenceHandler { @@ -50,11 +51,25 @@ impl offence::OnOffenceHandler for OnOff _offenders: &[OffenceDetails], slash_fraction: &[Perbill], _offence_session: SessionIndex, - ) { - ON_OFFENCE_PERBILL.with(|f| { - *f.borrow_mut() = slash_fraction.to_vec(); - }); + ) -> Result<(), ()> { + if >::can_report() { + ON_OFFENCE_PERBILL.with(|f| { + *f.borrow_mut() = slash_fraction.to_vec(); + }); + + Ok(()) + } else { + Err(()) + } } + + fn can_report() -> bool { + CAN_REPORT.with(|c| *c.borrow()) + } +} + +pub fn set_can_report(can_report: bool) { + CAN_REPORT.with(|c| *c.borrow_mut() = can_report); } pub fn with_on_offence_fractions) -> R>(f: F) -> R { @@ -113,7 +128,9 @@ impl_outer_event! { pub fn new_test_ext() -> sp_io::TestExternalities { let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } /// Offences module. diff --git a/frame/offences/src/tests.rs b/frame/offences/src/tests.rs index 7e9c3986ed7dbb65add26dc9b084fa0265ac6b77..3179a0752318f1340ae924599734423b793fb012 100644 --- a/frame/offences/src/tests.rs +++ b/frame/offences/src/tests.rs @@ -21,9 +21,10 @@ use super::*; use crate::mock::{ Offences, System, Offence, TestEvent, KIND, new_test_ext, with_on_offence_fractions, - offence_reports, + offence_reports, set_can_report, }; use sp_runtime::Perbill; +use frame_support::traits::OnInitialize; use frame_system::{EventRecord, Phase}; #[test] @@ -130,7 +131,7 @@ fn should_deposit_event() { System::events(), vec![EventRecord { phase: Phase::Initialization, - event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode())), + event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode(), true)), topics: vec![], }] ); @@ -165,7 +166,7 @@ fn doesnt_deposit_event_for_dups() { System::events(), vec![EventRecord { phase: Phase::Initialization, - event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode())), + event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode(), true)), topics: vec![], }] ); @@ -212,3 +213,54 @@ fn should_properly_count_offences() { ); }); } + +#[test] +fn should_queue_and_resubmit_rejected_offence() { + new_test_ext().execute_with(|| { + set_can_report(false); + + // will get deferred + let offence = Offence { + validator_set_count: 5, + time_slot: 42, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + assert_eq!(Offences::deferred_offences().len(), 1); + // event also indicates unapplied. + assert_eq!( + System::events(), + vec![EventRecord { + phase: Phase::Initialization, + event: TestEvent::offences(crate::Event::Offence(KIND, 42u128.encode(), false)), + topics: vec![], + }] + ); + + // will not dequeue + Offences::on_initialize(2); + + // again + let offence = Offence { + validator_set_count: 5, + time_slot: 62, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + assert_eq!(Offences::deferred_offences().len(), 2); + + set_can_report(true); + + // can be submitted + let offence = Offence { + validator_set_count: 5, + time_slot: 72, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + assert_eq!(Offences::deferred_offences().len(), 2); + + Offences::on_initialize(3); + assert_eq!(Offences::deferred_offences().len(), 0); + }) +} diff --git a/frame/randomness-collective-flip/Cargo.toml b/frame/randomness-collective-flip/Cargo.toml index dd897cc71fa44a8de2894469cb0096f7aa3bf1e4..acd1c216884c036715f0f52b997cb0b78d5c6669 100644 --- a/frame/randomness-collective-flip/Cargo.toml +++ b/frame/randomness-collective-flip/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME randomness collective flip pallet" [dependencies] safe-mix = { version = "1.0", default-features = false } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } @@ -30,3 +30,6 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index 0ded7dd6b0c64e5c932cf42d2f7478dea1ef4723..fdc465b4dc3bb0e9163bc750595a290138a7b24d 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -35,12 +35,13 @@ //! ### Example - Get random seed for the current block //! //! ``` -//! use frame_support::{decl_module, dispatch, traits::Randomness}; +//! use frame_support::{decl_module, dispatch, traits::Randomness, weights::SimpleDispatchInfo}; //! //! pub trait Trait: frame_system::Trait {} //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = SimpleDispatchInfo::default()] //! pub fn random_module_example(origin) -> dispatch::DispatchResult { //! let _random_seed = >::random_seed(); //! Ok(()) @@ -54,7 +55,10 @@ use sp_std::{prelude::*, convert::TryInto}; use sp_runtime::traits::Hash; -use frame_support::{decl_module, decl_storage, traits::Randomness}; +use frame_support::{ + decl_module, decl_storage, traits::Randomness, + weights::{Weight, SimpleDispatchInfo, WeighData} +}; use safe_mix::TripletMix; use codec::Encode; use frame_system::Trait; @@ -69,7 +73,7 @@ fn block_number_to_index(block_number: T::BlockNumber) -> usize { decl_module! { pub struct Module for enum Call where origin: T::Origin { - fn on_initialize(block_number: T::BlockNumber) { + fn on_initialize(block_number: T::BlockNumber) -> Weight { let parent_hash = >::parent_hash(); >::mutate(|ref mut values| if values.len() < RANDOM_MATERIAL_LEN as usize { @@ -78,6 +82,8 @@ decl_module! { let index = block_number_to_index::(block_number); values[index] = parent_hash; }); + + SimpleDispatchInfo::default().weigh_data(()) } } } @@ -156,9 +162,11 @@ mod tests { use sp_runtime::{ Perbill, testing::Header, - traits::{BlakeTwo256, OnInitialize, Header as _, IdentityLookup}, + traits::{BlakeTwo256, Header as _, IdentityLookup}, + }; + use frame_support::{ + impl_outer_origin, parameter_types, weights::Weight, traits::{Randomness, OnInitialize}, }; - use frame_support::{impl_outer_origin, parameter_types, weights::Weight, traits::Randomness}; #[derive(Clone, PartialEq, Eq)] pub struct Test; diff --git a/frame/recovery/Cargo.toml b/frame/recovery/Cargo.toml index 35fc394880c77ab36add8c2e26f99119bedee43a..3347014f6e8cafc53a28626aad5124e35fa25b6a 100644 --- a/frame/recovery/Cargo.toml +++ b/frame/recovery/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME account recovery pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -33,3 +33,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/recovery/src/lib.rs b/frame/recovery/src/lib.rs index d7f5127ee3fc4ca28f90c550c67e5dc4c43648dd..c055f2bd97c67777893d493feb4d51c3934048ed 100644 --- a/frame/recovery/src/lib.rs +++ b/frame/recovery/src/lib.rs @@ -161,6 +161,7 @@ use frame_support::{ decl_module, decl_event, decl_storage, decl_error, ensure, Parameter, RuntimeDebug, weights::{GetDispatchInfo, SimpleDispatchInfo, FunctionOf}, traits::{Currency, ReservableCurrency, Get, BalanceStatus}, + dispatch::PostDispatchInfo, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -178,7 +179,7 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// The overarching call type. - type Call: Parameter + Dispatchable + GetDispatchInfo; + type Call: Parameter + Dispatchable + GetDispatchInfo; /// The currency mechanism. type Currency: ReservableCurrency; @@ -348,6 +349,7 @@ decl_module! { let target = Self::proxy(&who).ok_or(Error::::NotAllowed)?; ensure!(&target == &account, Error::::NotAllowed); call.dispatch(frame_system::RawOrigin::Signed(account).into()) + .map(|_| ()).map_err(|e| e.error) } /// Allow ROOT to bypass the recovery process and set an a rescuer account @@ -645,6 +647,7 @@ decl_module! { /// # /// - One storage mutation to check account is recovered by `who`. O(1) /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn cancel_recovered(origin, account: T::AccountId) { let who = ensure_signed(origin)?; // Check `who` is allowed to make a call on behalf of `account` diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index a5b7731c2286a3c4af60ee3ec54f64060c0953a5..9327ece572212af48e739eac3725925592c9bc95 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -21,12 +21,13 @@ use super::*; use frame_support::{ impl_outer_origin, impl_outer_dispatch, impl_outer_event, parameter_types, weights::Weight, + traits::{OnInitialize, OnFinalize}, }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ - Perbill, traits::{BlakeTwo256, IdentityLookup, OnInitialize, OnFinalize}, testing::Header, + Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header, }; use crate as recovery; diff --git a/frame/recovery/src/tests.rs b/frame/recovery/src/tests.rs index 9c644291c906dcbf00ab0185d3e1065fc8d54fb1..fb993043a5b005f4e94976f99a9701af6ab83b79 100644 --- a/frame/recovery/src/tests.rs +++ b/frame/recovery/src/tests.rs @@ -240,7 +240,7 @@ fn initiate_recovery_works() { assert_eq!(Balances::reserved_balance(1), 10); // Recovery status object is created correctly let recovery_status = ActiveRecovery { - created: 1, + created: 0, deposit: 10, friends: vec![], }; @@ -288,7 +288,7 @@ fn vouch_recovery_works() { assert_ok!(Recovery::vouch_recovery(Origin::signed(3), 5, 1)); // Final recovery status object is updated correctly let recovery_status = ActiveRecovery { - created: 1, + created: 0, deposit: 10, friends: vec![2, 3, 4], }; diff --git a/frame/scheduler/Cargo.toml b/frame/scheduler/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..531de95867bbbaace5557f7b0e2d7e3d057088b8 --- /dev/null +++ b/frame/scheduler/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "pallet-scheduler" +version = "2.0.0-alpha.5" +authors = ["Parity Technologies "] +edition = "2018" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet" + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +frame-benchmarking = { version = "2.0.0-alpha.4", default-features = false, path = "../benchmarking" } +frame-support = { version = "2.0.0-alpha.4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.4", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/io" } + +[dev-dependencies] +sp-core = { version = "2.0.0-alpha.4", path = "../../primitives/core", default-features = false } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-runtime/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "sp-io/std", + "sp-std/std" +] diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..f70204cb1a6b7201c89173cbe0dba6d5f10236ab --- /dev/null +++ b/frame/scheduler/src/lib.rs @@ -0,0 +1,522 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Scheduler +//! +//! \# Scheduler +//! +//! - \[`scheduler::Trait`](./trait.Trait.html) +//! - \[`Call`](./enum.Call.html) +//! - \[`Module`](./struct.Module.html) +//! +//! \## Overview +//! +//! // Short description of pallet's purpose. +//! // Links to Traits that should be implemented. +//! // What this pallet is for. +//! // What functionality the pallet provides. +//! // When to use the pallet (use case examples). +//! // How it is used. +//! // Inputs it uses and the source of each input. +//! // Outputs it produces. +//! +//! \## Terminology +//! +//! \## Goals +//! +//! \## Interface +//! +//! \### Dispatchable Functions + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use codec::{Encode, Decode}; +use sp_runtime::{RuntimeDebug, traits::{Zero, One}}; +use frame_support::{ + dispatch::{Dispatchable, DispatchResult, Parameter}, decl_module, decl_storage, decl_event, + traits::{Get, schedule}, + weights::{GetDispatchInfo, Weight}, +}; +use frame_system::{self as system}; + +/// Our pallet's configuration trait. All our types and constants go in here. If the +/// pallet is dependent on specific other pallets, then their configuration traits +/// should be added to our implied traits list. +/// +/// `system::Trait` should always be included in our implied traits. +pub trait Trait: system::Trait { + /// The overarching event type. + type Event: From> + Into<::Event>; + + /// The aggregated origin which the dispatch will take. + type Origin: From>; + + /// The aggregated call type. + type Call: Parameter + Dispatchable::Origin> + GetDispatchInfo; + + /// The maximum weight that may be scheduled per block for any dispatchables of less priority + /// than `schedule::HARD_DEADLINE`. + type MaximumWeight: Get; +} + +/// Just a simple index for naming period tasks. +pub type PeriodicIndex = u32; +/// The location of a scheduled task that can be used to remove it. +pub type TaskAddress = (BlockNumber, u32); + +/// Information regarding an item to be executed in the future. +#[derive(Clone, RuntimeDebug, Encode, Decode)] +pub struct Scheduled { + /// The unique identity for this task, if there is one. + maybe_id: Option>, + /// This task's priority. + priority: schedule::Priority, + /// The call to be dispatched. + call: Call, + /// If the call is periodic, then this points to the information concerning that. + maybe_periodic: Option>, +} + +decl_storage! { + trait Store for Module as Scheduler { + /// Items to be executed, indexed by the block number that they should be executed on. + pub Agenda: map hasher(twox_64_concat) T::BlockNumber + => Vec::Call, T::BlockNumber>>>; + + /// Lookup from identity to the block number and index of the task. + Lookup: map hasher(twox_64_concat) Vec => Option>; + } +} + +decl_event!( + pub enum Event where ::BlockNumber { + Scheduled(BlockNumber), + Dispatched(TaskAddress, Option>, DispatchResult), + } +); + +decl_module! { + // Simple declaration of the `Module` type. Lets the macro know what its working on. + pub struct Module for enum Call where origin: ::Origin { + fn deposit_event() = default; + + fn on_initialize(now: T::BlockNumber) -> Weight { + let limit = T::MaximumWeight::get(); + let mut queued = Agenda::::take(now).into_iter() + .enumerate() + .filter_map(|(index, s)| s.map(|inner| (index as u32, inner))) + .collect::>(); + queued.sort_by_key(|(_, s)| s.priority); + let mut result = 0; + let unused_items = queued.into_iter() + .enumerate() + .scan(0, |cumulative_weight, (order, (index, s))| { + *cumulative_weight += s.call.get_dispatch_info().weight; + Some((order, index, *cumulative_weight, s)) + }) + .filter_map(|(order, index, cumulative_weight, mut s)| { + if s.priority <= schedule::HARD_DEADLINE || cumulative_weight <= limit || order == 0 { + let r = s.call.clone().dispatch(system::RawOrigin::Root.into()); + let maybe_id = s.maybe_id.clone(); + if let &Some((period, count)) = &s.maybe_periodic { + if count > 1 { + s.maybe_periodic = Some((period, count - 1)); + } else { + s.maybe_periodic = None; + } + let next = now + period; + if let Some(ref id) = s.maybe_id { + let next_index = Agenda::::decode_len(now + period).unwrap_or(0) as u32; + Lookup::::insert(id, (next, next_index)); + } + Agenda::::append_or_insert(next, &[Some(s)][..]); + } else { + if let Some(ref id) = s.maybe_id { + Lookup::::remove(id); + } + } + Self::deposit_event(RawEvent::Dispatched( + (now, index), + maybe_id, + r.map(|_| ()).map_err(|e| e.error) + )); + result = cumulative_weight; + None + } else { + Some(Some(s)) + } + }) + .collect::>(); + if !unused_items.is_empty() { + let next = now + One::one(); + Agenda::::append_or_insert(next, &unused_items[..]); + } + result + } + } +} + +impl schedule::Anon::Call> for Module { + type Address = TaskAddress; + + fn schedule( + when: T::BlockNumber, + maybe_periodic: Option>, + priority: schedule::Priority, + call: ::Call + ) -> Self::Address { + // sanitize maybe_periodic + let maybe_periodic = maybe_periodic + .filter(|p| p.1 > 1 && !p.0.is_zero()) + // Remove one from the number of repetitions since we will schedule one now. + .map(|(p, c)| (p, c - 1)); + let s = Some(Scheduled { maybe_id: None, priority, call, maybe_periodic }); + Agenda::::append_or_insert(when, &[s][..]); + (when, Agenda::::decode_len(when).unwrap_or(1) as u32 - 1) + } + + fn cancel((when, index): Self::Address) -> Result<(), ()> { + if let Some(s) = Agenda::::mutate(when, |agenda| agenda.get_mut(index as usize).and_then(Option::take)) { + if let Some(id) = s.maybe_id { + Lookup::::remove(id) + } + Ok(()) + } else { + Err(()) + } + } +} + +impl schedule::Named::Call> for Module { + type Address = TaskAddress; + + fn schedule_named( + id: impl Encode, + when: T::BlockNumber, + maybe_periodic: Option>, + priority: schedule::Priority, + call: ::Call, + ) -> Result { + // determine id and ensure it is unique + let id = id.encode(); + if Lookup::::contains_key(&id) { + return Err(()) + } + + // sanitize maybe_periodic + let maybe_periodic = maybe_periodic + .filter(|p| p.1 > 1 && !p.0.is_zero()) + // Remove one from the number of repetitions since we will schedule one now. + .map(|(p, c)| (p, c - 1)); + + let s = Scheduled { maybe_id: Some(id.clone()), priority, call, maybe_periodic }; + Agenda::::append_or_insert(when, &[Some(s)][..]); + let index = Agenda::::decode_len(when).unwrap_or(1) as u32 - 1; + let address = (when, index); + Lookup::::insert(&id, &address); + Ok(address) + } + + fn cancel_named(id: impl Encode) -> Result<(), ()> { + if let Some((when, index)) = id.using_encoded(|d| Lookup::::take(d)) { + let i = index as usize; + Agenda::::mutate(when, |agenda| if let Some(s) = agenda.get_mut(i) { *s = None }); + Ok(()) + } else { + Err(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use frame_support::{ + impl_outer_event, impl_outer_origin, impl_outer_dispatch, parameter_types, assert_ok, + traits::{OnInitialize, OnFinalize, schedule::{Anon, Named}}, + weights::{DispatchClass, FunctionOf} + }; + use sp_core::H256; + // The testing primitives are very useful for avoiding having to work with signatures + // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. + use sp_runtime::{ + Perbill, + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, + }; + use crate as scheduler; + + mod logger { + use super::*; + use std::cell::RefCell; + use frame_system::ensure_root; + + thread_local! { + static LOG: RefCell> = RefCell::new(Vec::new()); + } + pub fn log() -> Vec { + LOG.with(|log| log.borrow().clone()) + } + pub trait Trait: system::Trait { + type Event: From + Into<::Event>; + } + decl_storage! { + trait Store for Module as Logger { + } + } + decl_event! { + pub enum Event { + Logged(u32, Weight), + } + } + decl_module! { + // Simple declaration of the `Module` type. Lets the macro know what its working on. + pub struct Module for enum Call where origin: ::Origin { + fn deposit_event() = default; + + #[weight = FunctionOf( + |args: (&u32, &Weight)| *args.1, + |_: (&u32, &Weight)| DispatchClass::Normal, + true + )] + fn log(origin, i: u32, weight: Weight) { + ensure_root(origin)?; + Self::deposit_event(Event::Logged(i, weight)); + LOG.with(|log| { + log.borrow_mut().push(i); + }) + } + } + } + } + + impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} + } + + impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + system::System, + logger::Logger, + } + } + + impl_outer_event! { + pub enum Event for Test { + system, + logger, + scheduler, + } + } + // For testing the pallet, we construct most of a mock runtime. This means + // first constructing a configuration type (`Test`) which `impl`s each of the + // configuration traits of pallets we want to use. + #[derive(Clone, Eq, PartialEq)] + pub struct Test; + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); + } + impl system::Trait for Test { + type Origin = Origin; + type Call = (); + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + } + impl logger::Trait for Test { + type Event = (); + } + parameter_types! { + pub const MaximumWeight: Weight = 10_000; + } + impl Trait for Test { + type Event = (); + type Origin = Origin; + type Call = Call; + type MaximumWeight = MaximumWeight; + } + type System = system::Module; + type Logger = logger::Module; + type Scheduler = Module; + + // This function basically just builds a genesis storage key/value store according to + // our desired mockup. + fn new_test_ext() -> sp_io::TestExternalities { + let t = system::GenesisConfig::default().build_storage::().unwrap(); + t.into() + } + + fn run_to_block(n: u64) { + while System::block_number() < n { + Scheduler::on_finalize(System::block_number()); + System::set_block_number(System::block_number() + 1); + Scheduler::on_initialize(System::block_number()); + } + } + + #[test] + fn basic_scheduling_works() { + new_test_ext().execute_with(|| { + Scheduler::schedule(4, None, 127, Call::Logger(logger::Call::log(42, 1000))); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![42u32]); + run_to_block(100); + assert_eq!(logger::log(), vec![42u32]); + }); + } + + #[test] + fn periodic_scheduling_works() { + new_test_ext().execute_with(|| { + // at #4, every 3 blocks, 3 times. + Scheduler::schedule(4, Some((3, 3)), 127, Call::Logger(logger::Call::log(42, 1000))); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![42u32]); + run_to_block(6); + assert_eq!(logger::log(), vec![42u32]); + run_to_block(7); + assert_eq!(logger::log(), vec![42u32, 42u32]); + run_to_block(9); + assert_eq!(logger::log(), vec![42u32, 42u32]); + run_to_block(10); + assert_eq!(logger::log(), vec![42u32, 42u32, 42u32]); + run_to_block(100); + assert_eq!(logger::log(), vec![42u32, 42u32, 42u32]); + }); + } + + #[test] + fn cancel_named_scheduling_works_with_normal_cancel() { + new_test_ext().execute_with(|| { + // at #4. + Scheduler::schedule_named(1u32, 4, None, 127, Call::Logger(logger::Call::log(69, 1000))).unwrap(); + let i = Scheduler::schedule(4, None, 127, Call::Logger(logger::Call::log(42, 1000))); + run_to_block(3); + assert!(logger::log().is_empty()); + assert_ok!(Scheduler::cancel_named(1u32)); + assert_ok!(Scheduler::cancel(i)); + run_to_block(100); + assert!(logger::log().is_empty()); + }); + } + + #[test] + fn cancel_named_periodic_scheduling_works() { + new_test_ext().execute_with(|| { + // at #4, every 3 blocks, 3 times. + Scheduler::schedule_named(1u32, 4, Some((3, 3)), 127, Call::Logger(logger::Call::log(42, 1000))).unwrap(); + // same id results in error. + assert!(Scheduler::schedule_named(1u32, 4, None, 127, Call::Logger(logger::Call::log(69, 1000))).is_err()); + // different id is ok. + Scheduler::schedule_named(2u32, 8, None, 127, Call::Logger(logger::Call::log(69, 1000))).unwrap(); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![42u32]); + run_to_block(6); + assert_ok!(Scheduler::cancel_named(1u32)); + run_to_block(100); + assert_eq!(logger::log(), vec![42u32, 69u32]); + }); + } + + #[test] + fn scheduler_respects_weight_limits() { + new_test_ext().execute_with(|| { + Scheduler::schedule(4, None, 127, Call::Logger(logger::Call::log(42, 6000))); + Scheduler::schedule(4, None, 127, Call::Logger(logger::Call::log(69, 6000))); + run_to_block(4); + assert_eq!(logger::log(), vec![42u32]); + run_to_block(5); + assert_eq!(logger::log(), vec![42u32, 69u32]); + }); + } + + #[test] + fn scheduler_respects_hard_deadlines_more() { + new_test_ext().execute_with(|| { + Scheduler::schedule(4, None, 0, Call::Logger(logger::Call::log(42, 6000))); + Scheduler::schedule(4, None, 0, Call::Logger(logger::Call::log(69, 6000))); + run_to_block(4); + assert_eq!(logger::log(), vec![42u32, 69u32]); + }); + } + + #[test] + fn scheduler_respects_priority_ordering() { + new_test_ext().execute_with(|| { + Scheduler::schedule(4, None, 1, Call::Logger(logger::Call::log(42, 6000))); + Scheduler::schedule(4, None, 0, Call::Logger(logger::Call::log(69, 6000))); + run_to_block(4); + assert_eq!(logger::log(), vec![69u32, 42u32]); + }); + } + + #[test] + fn scheduler_respects_priority_ordering_with_soft_deadlines() { + new_test_ext().execute_with(|| { + Scheduler::schedule(4, None, 255, Call::Logger(logger::Call::log(42, 5000))); + Scheduler::schedule(4, None, 127, Call::Logger(logger::Call::log(69, 5000))); + Scheduler::schedule(4, None, 126, Call::Logger(logger::Call::log(2600, 6000))); + run_to_block(4); + assert_eq!(logger::log(), vec![2600u32]); + run_to_block(5); + assert_eq!(logger::log(), vec![2600u32, 69u32, 42u32]); + }); + } + + #[test] + fn initialize_weight_is_correct() { + new_test_ext().execute_with(|| { + Scheduler::schedule(1, None, 255, Call::Logger(logger::Call::log(3, 1000))); + Scheduler::schedule(1, None, 128, Call::Logger(logger::Call::log(42, 5000))); + Scheduler::schedule(1, None, 127, Call::Logger(logger::Call::log(69, 5000))); + Scheduler::schedule(1, None, 126, Call::Logger(logger::Call::log(2600, 6000))); + let weight = Scheduler::on_initialize(1); + assert_eq!(weight, 6000); + let weight = Scheduler::on_initialize(2); + assert_eq!(weight, 10000); + let weight = Scheduler::on_initialize(3); + assert_eq!(weight, 1000); + let weight = Scheduler::on_initialize(4); + assert_eq!(weight, 0); + }); + } +} diff --git a/frame/scored-pool/Cargo.toml b/frame/scored-pool/Cargo.toml index a00b80be32c82510d6b450ec1d1b7015e46c449f..b878c5bb47559d3d8eab8cb842d8eb00182abed7 100644 --- a/frame/scored-pool/Cargo.toml +++ b/frame/scored-pool/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for scored pools" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -32,3 +32,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/scored-pool/src/lib.rs b/frame/scored-pool/src/lib.rs index ebabe791055c62f1c18ad84fb97b30359e84ecc4..2602d389626a702a2fea4e156e84e3fd52ad9d3e 100644 --- a/frame/scored-pool/src/lib.rs +++ b/frame/scored-pool/src/lib.rs @@ -61,6 +61,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] //! pub fn candidate(origin) -> dispatch::DispatchResult { //! let who = ensure_signed(origin)?; //! @@ -95,11 +96,12 @@ use sp_std::{ }; use frame_support::{ decl_module, decl_storage, decl_event, ensure, decl_error, - traits::{ChangeMembers, InitializeMembers, Currency, Get, ReservableCurrency}, + traits::{EnsureOrigin, ChangeMembers, InitializeMembers, Currency, Get, ReservableCurrency}, + weights::{Weight, SimpleDispatchInfo, WeighData}, }; use frame_system::{self as system, ensure_root, ensure_signed}; use sp_runtime::{ - traits::{EnsureOrigin, AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup}, + traits::{AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup}, }; type BalanceOf = <>::Currency as Currency<::AccountId>>::Balance; @@ -189,8 +191,8 @@ decl_storage! { >::insert(who, true); }); - /// Sorts the `Pool` by score in a descending order. Entities which - /// have a score of `None` are sorted to the beginning of the vec. + // Sorts the `Pool` by score in a descending order. Entities which + // have a score of `None` are sorted to the beginning of the vec. pool.sort_by_key(|(_, maybe_score)| Reverse(maybe_score.unwrap_or_default()) ); @@ -245,11 +247,12 @@ decl_module! { /// Every `Period` blocks the `Members` set is refreshed from the /// highest scoring members in the pool. - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { if n % T::Period::get() == Zero::zero() { let pool = >::get(); >::refresh_members(pool, ChangeReceiver::MembershipChanged); } + SimpleDispatchInfo::default().weigh_data(()) } /// Add `origin` to the pool of candidates. @@ -263,6 +266,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the transactor in the `Pool`. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn submit_candidacy(origin) { let who = ensure_signed(origin)?; ensure!(!>::contains_key(&who), Error::::AlreadyInPool); @@ -292,6 +296,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the transactor in the `Pool`. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn withdraw_candidacy( origin, index: u32 @@ -311,6 +316,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of `dest` in the `Pool`. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn kick( origin, dest: ::Source, @@ -335,6 +341,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the `dest` in the `Pool`. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn score( origin, dest: ::Source, @@ -375,6 +382,7 @@ decl_module! { /// (this happens each `Period`). /// /// May only be called from root. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn change_member_count(origin, count: u32) { ensure_root(origin)?; >::put(&count); diff --git a/frame/scored-pool/src/tests.rs b/frame/scored-pool/src/tests.rs index 4b21339505ded6b55d9a520595af2df1c8277fca..8d87a20f757b223443b2b134fb1d66f0ab793b53 100644 --- a/frame/scored-pool/src/tests.rs +++ b/frame/scored-pool/src/tests.rs @@ -19,8 +19,8 @@ use super::*; use mock::*; -use frame_support::{assert_ok, assert_noop}; -use sp_runtime::traits::{OnInitialize, BadOrigin}; +use frame_support::{assert_ok, assert_noop, traits::OnInitialize}; +use sp_runtime::traits::BadOrigin; type ScoredPool = Module; type System = frame_system::Module; diff --git a/frame/session/Cargo.toml b/frame/session/Cargo.toml index 2e1e9e2433aa79015ed6415fb578ea5e182a60a7..f12a8b4a71f2fc8df7f81ac50098c32ffd9cd7aa 100644 --- a/frame/session/Cargo.toml +++ b/frame/session/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME sessions pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } @@ -40,3 +40,6 @@ std = [ "sp-trie/std", "sp-io/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/session/benchmarking/Cargo.toml b/frame/session/benchmarking/Cargo.toml index 434a9e1ec55b88a94cc92e6b3781694b96e5570f..140116c82c640568f8683b0ed9309ab51088d672 100644 --- a/frame/session/benchmarking/Cargo.toml +++ b/frame/session/benchmarking/Cargo.toml @@ -13,9 +13,19 @@ sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../.. sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../../system" } frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../../benchmarking" } +frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../support" } pallet-staking = { version = "2.0.0-alpha.5", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../session" } +[dev-dependencies] +serde = { version = "1.0.101" } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } +pallet-staking-reward-curve = { version = "2.0.0-alpha.5", path = "../../staking/reward-curve" } +sp-io ={ path = "../../../primitives/io", version = "2.0.0-alpha.5"} +pallet-timestamp = { version = "2.0.0-alpha.5", path = "../../timestamp" } +pallet-balances = { version = "2.0.0-alpha.5", path = "../../balances" } + [features] default = ["std"] std = [ @@ -23,6 +33,10 @@ std = [ "sp-runtime/std", "frame-system/std", "frame-benchmarking/std", + "frame-support/std", "pallet-staking/std", "pallet-session/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index db925bd72ee45e204eca8faeee4a97d0d4d6a09d..3b91c2fdc5194f56086b139eb33a71efc468e050 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -19,6 +19,8 @@ #![cfg_attr(not(feature = "std"), no_std)] +mod mock; + use sp_std::prelude::*; use sp_std::vec; @@ -42,16 +44,33 @@ benchmarks! { set_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let validator = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; - }: _(RawOrigin::Signed(validator), keys, proof) + }: _(RawOrigin::Signed(v_controller), keys, proof) purge_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let validator = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; - Session::::set_keys(RawOrigin::Signed(validator.clone()).into(), keys, proof)?; - }: _(RawOrigin::Signed(validator)) + Session::::set_keys(RawOrigin::Signed(v_controller.clone()).into(), keys, proof)?; + }: _(RawOrigin::Signed(v_controller)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set_keys::()); + assert_ok!(test_benchmark_purge_keys::()); + }); + } } diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff7965efcacebbe6e7dd64a5999f9fcf196664cb --- /dev/null +++ b/frame/session/benchmarking/src/mock.rs @@ -0,0 +1,186 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Mock file for staking fuzzing. + +#![cfg(test)] + +use sp_runtime::traits::{Convert, SaturatedConversion, IdentityLookup}; +use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types}; + +type AccountId = u64; +type AccountIndex = u32; +type BlockNumber = u64; +type Balance = u64; + +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Staking = pallet_staking::Module; +type Session = pallet_session::Module; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + pallet_staking::Staking, + } +} + +pub struct CurrencyToVoteHandler; +impl Convert for CurrencyToVoteHandler { + fn convert(x: u64) -> u64 { + x + } +} +impl Convert for CurrencyToVoteHandler { + fn convert(x: u128) -> u64 { + x.saturated_into() + } +} + +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct Test; + +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = AccountIndex; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = sp_core::H256; + type Hashing = ::sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = sp_runtime::testing::Header; + type Event = (); + type BlockHashCount = (); + type MaximumBlockWeight = (); + type AvailableBlockRatio = (); + type MaximumBlockLength = (); + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (Balances,); +} +parameter_types! { + pub const ExistentialDeposit: Balance = 10; +} +impl pallet_balances::Trait for Test { + type Balance = Balance; + type Event = (); + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} + +parameter_types! { + pub const MinimumPeriod: u64 = 5; +} +impl pallet_timestamp::Trait for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; +} +impl pallet_session::historical::Trait for Test { + type FullIdentification = pallet_staking::Exposure; + type FullIdentificationOf = pallet_staking::ExposureOf; +} + +sp_runtime::impl_opaque_keys! { + pub struct SessionKeys { + pub foo: sp_runtime::testing::UintAuthorityId, + } +} + +pub struct TestSessionHandler; +impl pallet_session::SessionHandler for TestSessionHandler { + const KEY_TYPE_IDS: &'static [sp_runtime::KeyTypeId] = &[]; + + fn on_genesis_session(_validators: &[(AccountId, Ks)]) {} + + fn on_new_session( + _: bool, + _: &[(AccountId, Ks)], + _: &[(AccountId, Ks)], + ) {} + + fn on_disabled(_: usize) {} +} + +impl pallet_session::Trait for Test { + type SessionManager = pallet_session::historical::NoteHistoricalRoot; + type Keys = SessionKeys; + type ShouldEndSession = pallet_session::PeriodicSessions<(), ()>; + type NextSessionRotation = pallet_session::PeriodicSessions<(), ()>; + type SessionHandler = TestSessionHandler; + type Event = (); + type ValidatorId = AccountId; + type ValidatorIdOf = pallet_staking::StashOf; + type DisabledValidatorsThreshold = (); +} +pallet_staking_reward_curve::build! { + const I_NPOS: sp_runtime::curve::PiecewiseLinear<'static> = curve!( + min_inflation: 0_025_000, + max_inflation: 0_100_000, + ideal_stake: 0_500_000, + falloff: 0_050_000, + max_piece_count: 40, + test_precision: 0_005_000, + ); +} +parameter_types! { + pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; + pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const UnsignedPriority: u64 = 1 << 20; +} + +pub type Extrinsic = sp_runtime::testing::TestXt; +type SubmitTransaction = frame_system::offchain::TransactionSubmitter< + sp_runtime::testing::UintAuthorityId, + Test, + Extrinsic, +>; + +impl pallet_staking::Trait for Test { + type Currency = Balances; + type UnixTime = pallet_timestamp::Module; + type CurrencyToVote = CurrencyToVoteHandler; + type RewardRemainder = (); + type Event = (); + type Slash = (); + type Reward = (); + type SessionsPerEra = (); + type SlashDeferDuration = (); + type SlashCancelOrigin = frame_system::EnsureRoot; + type BondingDuration = (); + type SessionInterface = Self; + type RewardCurve = RewardCurve; + type NextNewSession = Session; + type ElectionLookahead = (); + type Call = Call; + type SubmitTransaction = SubmitTransaction; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = UnsignedPriority; +} + +impl crate::Trait for Test {} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + sp_io::TestExternalities::new(t) +} diff --git a/frame/session/src/historical.rs b/frame/session/src/historical.rs index 12b88b347c671451dd4e4bb28b941614c580ba13..f9990dd1e8a7a716f4a557467fe5364602c45fe3 100644 --- a/frame/session/src/historical.rs +++ b/frame/session/src/historical.rs @@ -310,12 +310,12 @@ impl> frame_support::traits::KeyOwnerProofSystem<(KeyTy mod tests { use super::*; use sp_core::crypto::key_types::DUMMY; - use sp_runtime::{traits::OnInitialize, testing::UintAuthorityId}; + use sp_runtime::testing::UintAuthorityId; use crate::mock::{ NEXT_VALIDATORS, force_new_session, set_next_validators, Test, System, Session, }; - use frame_support::traits::KeyOwnerProofSystem; + use frame_support::traits::{KeyOwnerProofSystem, OnInitialize}; type Historical = Module; diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index f4490e50a097f6fb6fe2901808b3453dd4387b61..9346b060fa48a8afd22d01dc1d88d6a6b743a8e4 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -102,12 +102,16 @@ use sp_std::{prelude::*, marker::PhantomData, ops::{Sub, Rem}}; use codec::Decode; use sp_runtime::{KeyTypeId, Perbill, RuntimeAppPublic, BoundToRuntimeAppPublic}; -use frame_support::weights::SimpleDispatchInfo; -use sp_runtime::traits::{Convert, Zero, Member, OpaqueKeys}; +use sp_runtime::traits::{Convert, Zero, Member, OpaqueKeys, Saturating}; use sp_staking::SessionIndex; -use frame_support::{ensure, decl_module, decl_event, decl_storage, decl_error, ConsensusEngineId}; -use frame_support::{traits::{Get, FindAuthor, ValidatorRegistration}, Parameter}; -use frame_support::dispatch::{self, DispatchResult, DispatchError}; +use frame_support::{ + ensure, decl_module, decl_event, decl_storage, decl_error, ConsensusEngineId, Parameter, + traits::{ + Get, FindAuthor, ValidatorRegistration, EstimateNextSessionRotation, EstimateNextNewSession, + }, + dispatch::{self, DispatchResult, DispatchError}, + weights::{Weight, SimpleDispatchInfo, WeighData}, +}; use frame_system::{self as system, ensure_signed}; #[cfg(test)] @@ -145,6 +149,29 @@ impl< } } +impl< + BlockNumber: Rem + Sub + Zero + PartialOrd + Saturating + Clone, + Period: Get, + Offset: Get, +> EstimateNextSessionRotation for PeriodicSessions { + fn estimate_next_session_rotation(now: BlockNumber) -> Option { + let offset = Offset::get(); + let period = Period::get(); + Some(if now > offset { + let block_after_last_session = (now.clone() - offset) % period.clone(); + if block_after_last_session > Zero::zero() { + now.saturating_add( + period.saturating_sub(block_after_last_session) + ) + } else { + Zero::zero() + } + } else { + offset + }) + } +} + /// A trait for managing creation of new validator set. pub trait SessionManager { /// Plan a new session, and optionally provide the new validator set. @@ -328,6 +355,11 @@ pub trait Trait: frame_system::Trait { /// Indicator for when to end the session. type ShouldEndSession: ShouldEndSession; + /// Something that can predict the next session rotation. This should typically come from the + /// same logical unit that provides [`ShouldEndSession`], yet, it gives a best effort estimate. + /// It is helpful to implement [`EstimateNextNewSession`]. + type NextSessionRotation: EstimateNextSessionRotation; + /// Handler for managing new session. type SessionManager: SessionManager; @@ -495,10 +527,12 @@ decl_module! { /// Called when a block is initialized. Will rotate session if it is the last /// block of the current session. - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { if T::ShouldEndSession::should_end_session(n) { Self::rotate_session(); } + + SimpleDispatchInfo::default().weigh_data(()) } } } @@ -731,3 +765,11 @@ impl> FindAuthor validators.get(i as usize).map(|k| k.clone()) } } + +impl EstimateNextNewSession for Module { + /// This session module always calls new_session and next_session at the same time, hence we + /// do a simple proxy and pass the function to next rotation. + fn estimate_next_new_session(now: T::BlockNumber) -> Option { + T::NextSessionRotation::estimate_next_session_rotation(now) + } +} diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 9d64285b900f8645d0f2d76cbe736bc7a908194f..a888dcfb28e992ba61881155fcc24d796cca085c 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -150,6 +150,16 @@ pub fn reset_before_session_end_called() { BEFORE_SESSION_END_CALLED.with(|b| *b.borrow_mut() = false); } +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + GenesisConfig:: { + keys: NEXT_VALIDATORS.with(|l| + l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() + ), + }.assimilate_storage(&mut t).unwrap(); + sp_io::TestExternalities::new(t) +} + #[derive(Clone, Eq, PartialEq)] pub struct Test; @@ -205,6 +215,7 @@ impl Trait for Test { type Keys = MockSessionKeys; type Event = (); type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = (); } #[cfg(feature = "historical")] diff --git a/frame/session/src/tests.rs b/frame/session/src/tests.rs index 052313f21d3484c33993fbe1c106441c62ab11e8..abfd9f738b614e2d9078ddfe02c07a1f3d862a32 100644 --- a/frame/session/src/tests.rs +++ b/frame/session/src/tests.rs @@ -17,25 +17,15 @@ // Tests for the Session Pallet use super::*; -use frame_support::assert_ok; +use frame_support::{traits::OnInitialize, assert_ok}; use sp_core::crypto::key_types::DUMMY; -use sp_runtime::{traits::OnInitialize, testing::UintAuthorityId}; +use sp_runtime::testing::UintAuthorityId; use mock::{ - NEXT_VALIDATORS, SESSION_CHANGED, TEST_SESSION_CHANGED, authorities, force_new_session, - set_next_validators, set_session_length, session_changed, Test, Origin, System, Session, - reset_before_session_end_called, before_session_end_called, + SESSION_CHANGED, TEST_SESSION_CHANGED, authorities, force_new_session, + set_next_validators, set_session_length, session_changed, Origin, System, Session, + reset_before_session_end_called, before_session_end_called, new_test_ext, }; -fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - GenesisConfig:: { - keys: NEXT_VALIDATORS.with(|l| - l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() - ), - }.assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t) -} - fn initialize_block(block: u64) { SESSION_CHANGED.with(|l| *l.borrow_mut() = false); System::set_block_number(block); diff --git a/frame/society/Cargo.toml b/frame/society/Cargo.toml index 5130536ffd0a8a729e09454963fec7d389bd95f4..be419fb63fe2d0800dde5802e9ce1cf02de52b14 100644 --- a/frame/society/Cargo.toml +++ b/frame/society/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME society pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -34,3 +34,10 @@ std = [ "frame-support/std", "frame-system/std", ] +runtime-benchmarks = [ + "sp-runtime/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs index 03c61ba5329fe15f7e874f464fac7aa50dafde71..2061c21d9c5f992cb6a6254918a9133d14483ca6 100644 --- a/frame/society/src/lib.rs +++ b/frame/society/src/lib.rs @@ -256,14 +256,14 @@ use codec::{Encode, Decode}; use sp_runtime::{Percent, ModuleId, RuntimeDebug, traits::{ StaticLookup, AccountIdConversion, Saturating, Zero, IntegerSquareRoot, Hash, - TrailingZeroInput, CheckedSub, EnsureOrigin + TrailingZeroInput, CheckedSub } }; use frame_support::{decl_error, decl_module, decl_storage, decl_event, ensure, dispatch::DispatchResult}; -use frame_support::weights::SimpleDispatchInfo; +use frame_support::weights::{SimpleDispatchInfo, Weight, WeighData}; use frame_support::traits::{ Currency, ReservableCurrency, Randomness, Get, ChangeMembers, BalanceStatus, - ExistenceRequirement::AllowDeath + ExistenceRequirement::AllowDeath, EnsureOrigin }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -1028,7 +1028,7 @@ decl_module! { Self::deposit_event(RawEvent::NewMaxMembers(max)); } - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { let mut members = vec![]; // Run a candidate/membership rotation @@ -1045,6 +1045,8 @@ decl_module! { } Self::rotate_challenge(&mut members); } + + SimpleDispatchInfo::default().weigh_data(()) } } } @@ -1143,6 +1145,12 @@ impl EnsureOrigin for EnsureFounder { (r, _) => Err(T::Origin::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> T::Origin { + let founder = Founder::::get().expect("society founder should exist"); + T::Origin::from(system::RawOrigin::Signed(founder)) + } } /// Pick an item at pseudo-random from the slice, given the `rng`. `None` iff the slice is empty. diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index 158f139df5673e9568e247122f40ed1d64c6b0c7..a66a5e6e047d44230b35f62fcf1a78043b5e1085 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -18,14 +18,16 @@ use super::*; -use frame_support::{impl_outer_origin, parameter_types, ord_parameter_types}; +use frame_support::{ + impl_outer_origin, parameter_types, ord_parameter_types, traits::{OnInitialize, OnFinalize} +}; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ Perbill, testing::Header, - traits::{BlakeTwo256, IdentityLookup, OnInitialize, OnFinalize}, + traits::{BlakeTwo256, IdentityLookup}, }; use frame_system::EnsureSignedBy; diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 82409b9cb4b920305ad04c00610e8e2ee3757207..aac5616e4ea18cd2ff64d9dc0a4fe0da46e24c00 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -10,8 +10,7 @@ description = "FRAME pallet staking" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } -sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/phragmen" } sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} @@ -21,24 +20,41 @@ frame-support = { version = "2.0.0-alpha.5", default-features = false, path = ". frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } pallet-session = { version = "2.0.0-alpha.5", features = ["historical"], path = "../session", default-features = false } pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../authorship" } +sp-application-crypto = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/application-crypto" } +static_assertions = "1.1.0" +# Optional imports for tesing-utils feature +pallet-indices = { version = "2.0.0-alpha.4", optional = true, path = "../indices", default-features = false } +sp-core = { version = "2.0.0-alpha.4", optional = true, path = "../../primitives/core", default-features = false } +rand = { version = "0.7.3", optional = true, default-features = false } + +# Optional imports for benchmarking frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } [dev-dependencies] sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } pallet-timestamp = { version = "2.0.0-alpha.5", path = "../timestamp" } pallet-staking-reward-curve = { version = "2.0.0-alpha.5", path = "../staking/reward-curve" } substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } frame-benchmarking = { version = "2.0.0-alpha.5", path = "../benchmarking" } rand_chacha = { version = "0.2" } +parking_lot = "0.10.0" +env_logger = "0.7.1" +hex = "0.4" [features] +testing-utils = [ + "std", + "pallet-indices/std", + "sp-core/std", + "rand/std", +] default = ["std"] std = [ "serde", - "sp-keyring", "codec/std", "sp-std/std", "sp-phragmen/std", @@ -49,9 +65,13 @@ std = [ "pallet-session/std", "frame-system/std", "pallet-authorship/std", + "sp-application-crypto/std", + "sp-core/std", ] runtime-benchmarks = [ "rand_chacha", "frame-benchmarking", - "frame-system/runtime-benchmarks", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/fuzz/.gitignore b/frame/staking/fuzz/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..572e03bdf321b6cc3a99488183436905cefd086d --- /dev/null +++ b/frame/staking/fuzz/.gitignore @@ -0,0 +1,4 @@ + +target +corpus +artifacts diff --git a/frame/staking/fuzz/Cargo.lock b/frame/staking/fuzz/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..f6e8cfa08d6675144979b3a5ef10eaa8f9dcf0d3 --- /dev/null +++ b/frame/staking/fuzz/Cargo.lock @@ -0,0 +1,2189 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "ahash" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" +dependencies = [ + "const-random", +] + +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +dependencies = [ + "memchr", +] + +[[package]] +name = "arbitrary" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16971f2f0ce65c5cf2a1546cc6a0af102ecb11e265ddaa9433fb3e5bfdf676a4" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17b52e737c40a7d75abca20b29a19a0eb7ba9fc72c5a72dd282a0a3c2c0dc35" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "base58" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitmask" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da9b3d9f6f585199287a473f4f8dfab6566cf827d15c00c219f53c645687ead" + +[[package]] +name = "bitvec" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +dependencies = [ + "arrayvec 0.4.12", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "bumpalo" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" + +[[package]] +name = "byte-slice-cast" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "c2-chacha" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +dependencies = [ + "ppv-lite86", +] + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clear_on_drop" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" +dependencies = [ + "cc", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "const-random" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" +dependencies = [ + "getrandom", + "proc-macro-hack", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle 1.0.0", +] + +[[package]] +name = "curve25519-dalek" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" +dependencies = [ + "byteorder", + "clear_on_drop", + "digest", + "rand_core 0.3.1", + "subtle 2.2.2", +] + +[[package]] +name = "curve25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" +dependencies = [ + "byteorder", + "digest", + "rand_core 0.5.1", + "subtle 2.2.2", + "zeroize 1.1.0", +] + +[[package]] +name = "derive_more" +version = "0.99.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.0-pre.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" +dependencies = [ + "clear_on_drop", + "curve25519-dalek 2.0.0", + "rand 0.7.3", + "sha2", +] + +[[package]] +name = "environmental" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" + +[[package]] +name = "failure" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fixed-hash" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" +dependencies = [ + "byteorder", + "libc", + "rand 0.7.3", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "frame-benchmarking" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "sp-api", + "sp-io", + "sp-runtime", + "sp-runtime-interface", + "sp-std", +] + +[[package]] +name = "frame-metadata" +version = "11.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "serde", + "sp-core", + "sp-std", +] + +[[package]] +name = "frame-support" +version = "2.0.0-alpha.3" +dependencies = [ + "bitmask", + "frame-metadata", + "frame-support-procedural", + "impl-trait-for-tuples", + "log", + "once_cell", + "parity-scale-codec", + "paste", + "serde", + "sp-arithmetic", + "sp-core", + "sp-inherents", + "sp-io", + "sp-runtime", + "sp-state-machine", + "sp-std", + "tracing", +] + +[[package]] +name = "frame-support-procedural" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support-procedural-tools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "frame-support-procedural-tools" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support-procedural-tools-derive", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "frame-support-procedural-tools-derive" +version = "2.0.0-alpha.3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "frame-system" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "impl-trait-for-tuples", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-version", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "futures" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" + +[[package]] +name = "futures-executor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" + +[[package]] +name = "futures-macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" + +[[package]] +name = "futures-task" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" + +[[package]] +name = "futures-util" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +dependencies = [ + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" +dependencies = [ + "ahash", + "autocfg 0.1.7", +] + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "hmac-drbg" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" +dependencies = [ + "digest", + "generic-array", + "hmac", +] + +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-serde" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-serde" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bbe9ea9b182f0fb1cabbd61f4ff9b7b7b9197955e95a7e4c27de5055eb29ff8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "integer-sqrt" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65877bf7d44897a473350b1046277941cee20b263397e90869c50b6e766088b" + +[[package]] +name = "js-sys" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cb931d43e71f560c81badb0191596562bafad2be06a3f9025b845c847c60df5" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" + +[[package]] +name = "libfuzzer-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb789afcc589a08928d1e466087445ab740a0f70a2ee23d9349a0e3723d65e1b" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libsecp256k1" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" +dependencies = [ + "arrayref", + "crunchy", + "digest", + "hmac-drbg", + "rand 0.7.3", + "sha2", + "subtle 2.2.2", + "typenum", +] + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "memory-db" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" +dependencies = [ + "ahash", + "hash-db", + "hashbrown", + "parity-util-mem", +] + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + +[[package]] +name = "merlin" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" +dependencies = [ + "byteorder", + "keccak", + "rand_core 0.4.2", + "zeroize 1.1.0", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.0.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +dependencies = [ + "autocfg 1.0.0", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +dependencies = [ + "autocfg 1.0.0", +] + +[[package]] +name = "once_cell" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +dependencies = [ + "parking_lot 0.9.0", +] + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "pallet-authorship" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-authorship", + "sp-core", + "sp-inherents", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-balances" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "parity-scale-codec", + "serde", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-indices" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-keyring", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-session" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "pallet-timestamp", + "parity-scale-codec", + "serde", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", + "sp-trie", +] + +[[package]] +name = "pallet-staking" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "pallet-authorship", + "pallet-indices", + "pallet-session", + "parity-scale-codec", + "rand 0.7.3", + "serde", + "sp-application-crypto", + "sp-core", + "sp-io", + "sp-phragmen", + "sp-runtime", + "sp-staking", + "sp-std", + "static_assertions", +] + +[[package]] +name = "pallet-staking-fuzz" +version = "0.0.0" +dependencies = [ + "frame-support", + "frame-system", + "libfuzzer-sys", + "pallet-balances", + "pallet-indices", + "pallet-session", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "rand 0.7.3", + "sp-core", + "sp-io", + "sp-phragmen", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-staking-reward-curve" +version = "2.0.0-alpha.3" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pallet-timestamp" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "parity-scale-codec", + "serde", + "sp-inherents", + "sp-runtime", + "sp-std", + "sp-timestamp", +] + +[[package]] +name = "parity-scale-codec" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" +dependencies = [ + "arrayvec 0.5.1", + "bitvec", + "byte-slice-cast", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parity-util-mem" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" +dependencies = [ + "cfg-if", + "impl-trait-for-tuples", + "parity-util-mem-derive", + "parking_lot 0.10.0", + "primitive-types", + "winapi", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +dependencies = [ + "lock_api", + "parking_lot_core 0.7.0", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec 1.3.0", + "winapi", +] + +[[package]] +name = "paste" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +dependencies = [ + "byteorder", + "crypto-mac", +] + +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" + +[[package]] +name = "primitive-types" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-serde 0.3.0", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" + +[[package]] +name = "proc-macro2" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.1", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +dependencies = [ + "c2-chacha", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "regex" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "schnorrkel" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" +dependencies = [ + "curve25519-dalek 1.2.3", + "failure", + "merlin", + "rand 0.6.5", + "rand_core 0.4.2", + "rand_os", + "sha2", + "subtle 2.2.2", + "zeroize 0.9.3", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "send_wrapper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0eddf2e8f50ced781f288c19f18621fa72a3779e3cb58dbf23b07469b0abeb4" + +[[package]] +name = "serde" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" + +[[package]] +name = "sp-api" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "parity-scale-codec", + "sp-api-proc-macro", + "sp-core", + "sp-runtime", + "sp-state-machine", + "sp-std", + "sp-version", +] + +[[package]] +name = "sp-api-proc-macro" +version = "2.0.0-alpha.3" +dependencies = [ + "blake2-rfc", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-application-crypto" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-std", +] + +[[package]] +name = "sp-arithmetic" +version = "2.0.0-alpha.3" +dependencies = [ + "integer-sqrt", + "num-traits", + "parity-scale-codec", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-authorship" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "sp-inherents", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-core" +version = "2.0.0-alpha.3" +dependencies = [ + "base58", + "blake2-rfc", + "byteorder", + "ed25519-dalek", + "hash-db", + "hash256-std-hasher", + "hex", + "impl-serde 0.3.0", + "lazy_static", + "libsecp256k1", + "log", + "num-traits", + "parity-scale-codec", + "parity-util-mem", + "parking_lot 0.10.0", + "primitive-types", + "rand 0.7.3", + "regex", + "rustc-hex", + "schnorrkel", + "serde", + "sha2", + "sp-debug-derive", + "sp-externalities", + "sp-runtime-interface", + "sp-std", + "sp-storage", + "substrate-bip39", + "tiny-bip39", + "tiny-keccak", + "twox-hash", + "wasmi", + "zeroize 1.1.0", +] + +[[package]] +name = "sp-debug-derive" +version = "2.0.0-alpha.3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-externalities" +version = "0.8.0-alpha.3" +dependencies = [ + "environmental", + "sp-std", + "sp-storage", +] + +[[package]] +name = "sp-inherents" +version = "2.0.0-alpha.3" +dependencies = [ + "derive_more", + "parity-scale-codec", + "parking_lot 0.10.0", + "sp-core", + "sp-std", +] + +[[package]] +name = "sp-io" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "libsecp256k1", + "log", + "parity-scale-codec", + "sp-core", + "sp-externalities", + "sp-runtime-interface", + "sp-state-machine", + "sp-std", + "sp-trie", + "sp-wasm-interface", +] + +[[package]] +name = "sp-keyring" +version = "2.0.0-alpha.3" +dependencies = [ + "lazy_static", + "sp-core", + "sp-runtime", + "strum", +] + +[[package]] +name = "sp-panic-handler" +version = "2.0.0-alpha.3" +dependencies = [ + "backtrace", + "log", +] + +[[package]] +name = "sp-phragmen" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "serde", + "sp-core", + "sp-phragmen-compact", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-phragmen-compact" +version = "2.0.0-dev" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-runtime" +version = "2.0.0-alpha.3" +dependencies = [ + "hash256-std-hasher", + "impl-trait-for-tuples", + "log", + "parity-scale-codec", + "parity-util-mem", + "paste", + "rand 0.7.3", + "serde", + "sp-application-crypto", + "sp-arithmetic", + "sp-core", + "sp-inherents", + "sp-io", + "sp-std", +] + +[[package]] +name = "sp-runtime-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "sp-externalities", + "sp-runtime-interface-proc-macro", + "sp-std", + "sp-wasm-interface", + "static_assertions", +] + +[[package]] +name = "sp-runtime-interface-proc-macro" +version = "2.0.0-alpha.3" +dependencies = [ + "Inflector", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-staking" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-state-machine" +version = "0.8.0-alpha.3" +dependencies = [ + "hash-db", + "log", + "num-traits", + "parity-scale-codec", + "parking_lot 0.10.0", + "rand 0.7.3", + "sp-core", + "sp-externalities", + "sp-panic-handler", + "sp-trie", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-std" +version = "2.0.0-alpha.3" + +[[package]] +name = "sp-storage" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-serde 0.2.3", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-timestamp" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-api", + "sp-inherents", + "sp-runtime", + "sp-std", + "wasm-timer", +] + +[[package]] +name = "sp-trie" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "memory-db", + "parity-scale-codec", + "sp-core", + "sp-std", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-version" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-serde 0.2.3", + "parity-scale-codec", + "serde", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-wasm-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-std", + "wasmi", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6138f8f88a16d90134763314e3fc76fa3ed6a7db4725d6acf9a3ef95a3188d22" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "substrate-bip39" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" +dependencies = [ + "hmac", + "pbkdf2", + "schnorrkel", + "sha2", +] + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + +[[package]] +name = "subtle" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" + +[[package]] +name = "syn" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tiny-bip39" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +dependencies = [ + "failure", + "hmac", + "once_cell", + "pbkdf2", + "rand 0.7.3", + "rustc-hash", + "sha2", + "unicode-normalization", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" +dependencies = [ + "serde", +] + +[[package]] +name = "tracing" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1721cc8cf7d770cc4257872507180f35a4797272f5962f24c806af9e7faf52ab" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbad39da2f9af1cae3016339ad7f2c7a9e870f12e8fd04c4fd7ef35b30c0d2b" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "trie-db" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" +dependencies = [ + "hash-db", + "hashbrown", + "log", + "rustc-hex", + "smallvec 1.3.0", +] + +[[package]] +name = "trie-root" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +dependencies = [ + "hash-db", +] + +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" + +[[package]] +name = "uint" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" +dependencies = [ + "byteorder", + "crunchy", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +dependencies = [ + "smallvec 1.3.0", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasm-bindgen" +version = "0.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3557c397ab5a8e347d434782bcd31fc1483d927a6826804cec05cc792ee2519d" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0da9c9a19850d3af6df1cb9574970b566d617ecfaf36eb0b706b6f3ef9bd2f8" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457414a91863c0ec00090dba537f88ab955d93ca6555862c29b6d860990b8a8a" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6fde1d36e75a714b5fe0cffbb78978f222ea6baebb726af13c78869fdb4205" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25bda4168030a6412ea8a047e27238cadf56f0e53516e1e83fec0a8b7c786f6d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc9f36ad51f25b0219a3d4d13b90eb44cd075dff8b6280cca015775d7acaddd8" + +[[package]] +name = "wasm-timer" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324c5e65a08699c9c4334ba136597ab22b85dccd4b65dd1e36ccf8f723a95b54" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.9.0", + "pin-utils", + "send_wrapper", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmi" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" +dependencies = [ + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", +] + +[[package]] +name = "web-sys" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721c6263e2c66fd44501cc5efbfa2b7dfa775d13e4ea38c46299646ed1f9c70a" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zeroize" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/frame/staking/fuzz/Cargo.toml b/frame/staking/fuzz/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a78fbf17dc8a3a1dd3c8572167d8d1d27692059a --- /dev/null +++ b/frame/staking/fuzz/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "pallet-staking-fuzz" +version = "0.0.0" +authors = ["Automatically generated"] +publish = false +edition = "2018" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.3" +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +pallet-staking = { version = "2.0.0-alpha.2", path = "..", features = ["testing-utils"] } +pallet-staking-reward-curve = { version = "2.0.0-alpha.2", path = "../reward-curve" } +pallet-session = { version = "2.0.0-alpha.2", path = "../../session" } +pallet-indices = { version = "2.0.0-alpha.2", path = "../../indices" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../../balances" } +pallet-timestamp = { version = "2.0.0-alpha.2", path = "../../timestamp" } +frame-system = { version = "2.0.0-alpha.2", path = "../../system" } +frame-support = { version = "2.0.0-alpha.2", path = "../../support" } +sp-std = { version = "2.0.0-alpha.2", path = "../../../primitives/std" } +sp-io ={ version = "2.0.0-alpha.2", path = "../../../primitives/io" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-phragmen = { version = "2.0.0-alpha.2", path = "../../../primitives/phragmen" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +rand = "0.7.3" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "submit_solution" +path = "fuzz_targets/submit_solution.rs" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/fuzz/fuzz_targets/mock.rs b/frame/staking/fuzz/fuzz_targets/mock.rs new file mode 100644 index 0000000000000000000000000000000000000000..4bb3437f92368b35b7bfd2ef4f4529214972e6ca --- /dev/null +++ b/frame/staking/fuzz/fuzz_targets/mock.rs @@ -0,0 +1,182 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Mock file for staking fuzzing. + +use sp_runtime::traits::{Convert, SaturatedConversion}; +use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types}; + +type AccountId = u64; +type AccountIndex = u32; +type BlockNumber = u64; +type Balance = u64; + +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Staking = pallet_staking::Module; +type Indices = pallet_indices::Module; +type Session = pallet_session::Module; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + staking::Staking, + } +} + +pub struct CurrencyToVoteHandler; +impl Convert for CurrencyToVoteHandler { + fn convert(x: u64) -> u64 { + x + } +} +impl Convert for CurrencyToVoteHandler { + fn convert(x: u128) -> u64 { + x.saturated_into() + } +} + +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct Test; + +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = AccountIndex; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = sp_core::H256; + type Hashing = ::sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = Indices; + type Header = sp_runtime::testing::Header; + type Event = (); + type BlockHashCount = (); + type MaximumBlockWeight = (); + type AvailableBlockRatio = (); + type MaximumBlockLength = (); + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (Balances,); +} +parameter_types! { + pub const ExistentialDeposit: Balance = 10; +} +impl pallet_balances::Trait for Test { + type Balance = Balance; + type Event = (); + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} +impl pallet_indices::Trait for Test { + type AccountIndex = AccountIndex; + type Event = (); + type Currency = Balances; + type Deposit = (); +} +parameter_types! { + pub const MinimumPeriod: u64 = 5; +} +impl pallet_timestamp::Trait for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; +} +impl pallet_session::historical::Trait for Test { + type FullIdentification = pallet_staking::Exposure; + type FullIdentificationOf = pallet_staking::ExposureOf; +} + +sp_runtime::impl_opaque_keys! { + pub struct SessionKeys { + pub foo: sp_runtime::testing::UintAuthorityId, + } +} + +pub struct TestSessionHandler; +impl pallet_session::SessionHandler for TestSessionHandler { + const KEY_TYPE_IDS: &'static [sp_runtime::KeyTypeId] = &[]; + + fn on_genesis_session(_validators: &[(AccountId, Ks)]) {} + + fn on_new_session( + _: bool, + _: &[(AccountId, Ks)], + _: &[(AccountId, Ks)], + ) {} + + fn on_disabled(_: usize) {} +} + +impl pallet_session::Trait for Test { + type SessionManager = pallet_session::historical::NoteHistoricalRoot; + type Keys = SessionKeys; + type ShouldEndSession = pallet_session::PeriodicSessions<(), ()>; + type NextSessionRotation = pallet_session::PeriodicSessions<(), ()>; + type SessionHandler = TestSessionHandler; + type Event = (); + type ValidatorId = AccountId; + type ValidatorIdOf = pallet_staking::StashOf; + type DisabledValidatorsThreshold = (); +} +pallet_staking_reward_curve::build! { + const I_NPOS: sp_runtime::curve::PiecewiseLinear<'static> = curve!( + min_inflation: 0_025_000, + max_inflation: 0_100_000, + ideal_stake: 0_500_000, + falloff: 0_050_000, + max_piece_count: 40, + test_precision: 0_005_000, + ); +} +parameter_types! { + pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; + pub const MaxNominatorRewardedPerValidator: u32 = 64; +} + +pub type Extrinsic = sp_runtime::testing::TestXt; +type SubmitTransaction = frame_system::offchain::TransactionSubmitter< + sp_runtime::testing::UintAuthorityId, + Test, + Extrinsic, +>; + +impl pallet_staking::Trait for Test { + type Currency = Balances; + type Time = pallet_timestamp::Module; + type CurrencyToVote = CurrencyToVoteHandler; + type RewardRemainder = (); + type Event = (); + type Slash = (); + type Reward = (); + type SessionsPerEra = (); + type SlashDeferDuration = (); + type SlashCancelOrigin = frame_system::EnsureRoot; + type BondingDuration = (); + type SessionInterface = Self; + type RewardCurve = RewardCurve; + type NextNewSession = Session; + type ElectionLookahead = (); + type Call = Call; + type SubmitTransaction = SubmitTransaction; + type KeyType = sp_runtime::testing::UintAuthorityId; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; +} diff --git a/frame/staking/fuzz/fuzz_targets/submit_solution.rs b/frame/staking/fuzz/fuzz_targets/submit_solution.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d1fcf1d7ea850dd033b89e41514cd672848ab24 --- /dev/null +++ b/frame/staking/fuzz/fuzz_targets/submit_solution.rs @@ -0,0 +1,130 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Fuzzing for staking pallet. + +#![no_main] +use libfuzzer_sys::fuzz_target; +use mock::Test; +use pallet_staking::testing_utils::{ + self, USER, get_seq_phragmen_solution, get_weak_solution, setup_chain_stakers, + set_validator_count, signed_account, +}; +use frame_support::assert_ok; +use sp_runtime::{traits::Dispatchable, DispatchError}; + +mod mock; + +#[repr(u32)] +#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] +enum Mode { + /// Initial submission. This will be rather cheap. + InitialSubmission, + /// A better submission that will replace the previous ones. This is the most expensive. + StrongerSubmission, + /// A weak submission that will be rejected. This will be rather cheap. + WeakerSubmission, +} + +pub fn new_test_ext() -> Result { + frame_system::GenesisConfig::default().build_storage::().map(Into::into) +} + +fuzz_target!(|do_reduce: bool| { + let ext = new_test_ext(); + let mode: Mode = unsafe { std::mem::transmute(testing_utils::random(0, 2)) }; + let num_validators = testing_utils::random(50, 500); + let num_nominators = testing_utils::random(200, 2000); + let edge_per_voter = testing_utils::random(1, 16); + let to_elect = testing_utils::random(10, num_validators); + + println!("+++ instance with params {} / {} / {} / {:?} / {}", + num_nominators, + num_validators, + edge_per_voter, + mode, + to_elect, + ); + + ext.unwrap_or_default().execute_with(|| { + // initial setup + set_validator_count::(to_elect); + setup_chain_stakers::( + num_validators, + num_nominators, + edge_per_voter, + ); + + println!("++ Chain setup done."); + + // stuff to submit + let (winners, compact, score) = match mode { + Mode::InitialSubmission => { + /* No need to setup anything */ + get_seq_phragmen_solution::(do_reduce) + }, + Mode::StrongerSubmission => { + let (winners, compact, score) = get_weak_solution::(false); + assert_ok!( + >::submit_election_solution( + signed_account::(USER), + winners, + compact, + score, + ) + ); + get_seq_phragmen_solution::(do_reduce) + }, + Mode::WeakerSubmission => { + let (winners, compact, score) = get_seq_phragmen_solution::(do_reduce); + assert_ok!( + >::submit_election_solution( + signed_account::(USER), + winners, + compact, + score, + ) + ); + get_weak_solution::(false) + } + }; + + println!("++ Submission ready."); + + // must have chosen correct number of winners. + assert_eq!(winners.len() as u32, >::validator_count()); + + // final call and origin + let call = pallet_staking::Call::::submit_election_solution( + winners, + compact, + score, + ); + let caller = signed_account::(USER); + + // actually submit + match mode { + Mode::WeakerSubmission => { + assert_eq!( + call.dispatch(caller.into()).unwrap_err(), + DispatchError::Module { index: 0, error: 11, message: Some("PhragmenWeakSubmission") }, + ); + }, + _ => assert_ok!(call.dispatch(caller.into())), + }; + }) +}); diff --git a/frame/staking/reward-curve/Cargo.toml b/frame/staking/reward-curve/Cargo.toml index 1f3fd436fa4c59c7bad1ece52982ad116df5f153..b3b749e96cd51fa050dfb6233f0c99eadf6ad3fb 100644 --- a/frame/staking/reward-curve/Cargo.toml +++ b/frame/staking/reward-curve/Cargo.toml @@ -19,3 +19,6 @@ proc-macro-crate = "0.1.4" [dev-dependencies] sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/reward-curve/src/log.rs b/frame/staking/reward-curve/src/log.rs index 1a25dbb98628bd1b526fea508158490945d2a6da..e0929a95970139450c0e953f6df7e8849e5ae727 100644 --- a/frame/staking/reward-curve/src/log.rs +++ b/frame/staking/reward-curve/src/log.rs @@ -14,13 +14,13 @@ pub fn log2(p: u32, q: u32) -> u32 { } let mut n = 0u32; - while !(p >= 2u32.pow(n)*q) || !(p < 2u32.pow(n+1)*q) { + while !(p >= (1u32 << n)*q) || !(p < (1u32 << (n+1))*q) { n += 1; } - assert!(p < 2u32.pow(n+1) * q); + assert!(p < (1u32 << (n+1)) * q); - let y_num: u32 = (p - 2u32.pow(n) * q).try_into().unwrap(); - let y_den: u32 = (p + 2u32.pow(n) * q).try_into().unwrap(); + let y_num: u32 = (p - (1u32 << n) * q).try_into().unwrap(); + let y_den: u32 = (p + (1u32 << n) * q).try_into().unwrap(); let _2_div_ln_2 = 2_885_390u32; diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 300e77bbda679f518db89b16bda92fb1e3826270..2686623aa1eae85655beaf9a917679d1e483319f 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -20,7 +20,7 @@ use super::*; use rand_chacha::{rand_core::{RngCore, SeedableRng}, ChaChaRng}; -use sp_runtime::traits::One; +use sp_runtime::traits::{Dispatchable, One}; use sp_io::hashing::blake2_256; use frame_system::RawOrigin; @@ -98,8 +98,8 @@ pub fn create_validators_with_nominators_for_era(v: u32, n: u32) -> Re Ok(()) } -// This function generates one validator being nominated by n nominators. -// It starts an era and creates pending payouts. +// This function generates one validator being nominated by n nominators, and returns +//the validator stash account. It also starts an era and creates pending payouts. pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> Result { let mut points_total = 0; let mut points_individual = Vec::new(); @@ -114,7 +114,7 @@ pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> R let stash_lookup: ::Source = T::Lookup::unlookup(v_stash.clone()); points_total += 10; - points_individual.push((v_stash, 10)); + points_individual.push((v_stash.clone(), 10)); // Give the validator n nominators, but keep total users in the system the same. for i in 0 .. upper_bound { @@ -144,62 +144,7 @@ pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> R let total_payout = T::Currency::minimum_balance() * 1000.into(); >::insert(current_era, total_payout); - Ok(v_controller) -} - -// This function generates one nominator nominating v validators. -// It starts an era and creates pending payouts. -pub fn create_nominator_with_validators(v: u32) -> Result<(T::AccountId, Vec), &'static str> { - let mut validators = Vec::new(); - let mut points_total = 0; - let mut points_individual = Vec::new(); - - MinimumValidatorCount::put(0); - - // Create v validators - let mut validator_lookups = Vec::new(); - for i in 0 .. v { - let (v_stash, v_controller) = create_stash_controller::(i)?; - let validator_prefs = ValidatorPrefs { - commission: Perbill::from_percent(50), - }; - Staking::::validate(RawOrigin::Signed(v_controller.clone()).into(), validator_prefs)?; - let stash_lookup: ::Source = T::Lookup::unlookup(v_stash.clone()); - - points_total += 10; - points_individual.push((v_stash.clone(), 10)); - validator_lookups.push(stash_lookup); - // Add to the list if it is less than the number we want the nominator to have - if validators.len() < v as usize { - validators.push(v_stash.clone()) - } - } - - // Create a nominator - let (_n_stash, n_controller) = create_stash_controller::(u32::max_value())?; - Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), validator_lookups)?; - - ValidatorCount::put(v); - - // Start a new Era - let new_validators = Staking::::new_era(SessionIndex::one()).unwrap(); - - assert!(new_validators.len() == v as usize); - - // Give Era Points - let reward = EraRewardPoints:: { - total: points_total, - individual: points_individual.into_iter().collect(), - }; - - let current_era = CurrentEra::get().unwrap(); - ErasRewardPoints::::insert(current_era, reward); - - // Create reward pool - let total_payout = T::Currency::minimum_balance() * 1000.into(); - >::insert(current_era, total_payout); - - Ok((n_controller, validators)) + Ok(v_stash) } benchmarks! { @@ -274,11 +219,11 @@ benchmarks! { let c in 0 .. 1000; }: _(RawOrigin::Root, c) - force_no_eras { let i in 1 .. 1; }: _(RawOrigin::Root) + force_no_eras { let i in 0 .. 1; }: _(RawOrigin::Root) - force_new_era {let i in 1 .. 1; }: _(RawOrigin::Root) + force_new_era {let i in 0 .. 1; }: _(RawOrigin::Root) - force_new_era_always { let i in 1 .. 1; }: _(RawOrigin::Root) + force_new_era_always { let i in 0 .. 1; }: _(RawOrigin::Root) // Worst case scenario, the list of invulnerables is very long. set_invulnerables { @@ -306,18 +251,12 @@ benchmarks! { let slash_indices: Vec = (0 .. s).collect(); }: _(RawOrigin::Root, era, slash_indices) - payout_validator { + payout_stakers { let n in 1 .. MAX_NOMINATIONS as u32; let validator = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; let current_era = CurrentEra::get().unwrap(); - }: _(RawOrigin::Signed(validator), current_era) - - payout_nominator { - let v in 0 .. MAX_NOMINATIONS as u32; - let (nominator, validators) = create_nominator_with_validators::(v)?; - let current_era = CurrentEra::get().unwrap(); - let find_nominator = validators.into_iter().map(|x| (x, 0)).collect(); - }: _(RawOrigin::Signed(nominator), current_era, find_nominator) + let caller = account("caller", n, SEED); + }: _(RawOrigin::Signed(caller), validator, current_era) rebond { let l in 1 .. 1000; @@ -386,23 +325,56 @@ benchmarks! { &mut NegativeImbalanceOf::::zero() ); } + + payout_all { + let v in 1 .. 10; + let n in 1 .. 100; + MinimumValidatorCount::put(0); + create_validators_with_nominators_for_era::(v, n)?; + // Start a new Era + let new_validators = Staking::::new_era(SessionIndex::one()).unwrap(); + assert!(new_validators.len() == v as usize); + + let current_era = CurrentEra::get().unwrap(); + let mut points_total = 0; + let mut points_individual = Vec::new(); + let mut payout_calls = Vec::new(); + + for validator in new_validators.iter() { + points_total += 10; + points_individual.push((validator.clone(), 10)); + payout_calls.push(Call::::payout_stakers(validator.clone(), current_era)) + } + + // Give Era Points + let reward = EraRewardPoints:: { + total: points_total, + individual: points_individual.into_iter().collect(), + }; + + ErasRewardPoints::::insert(current_era, reward); + + // Create reward pool + let total_payout = T::Currency::minimum_balance() * 1000.into(); + >::insert(current_era, total_payout); + + let caller: T::AccountId = account("caller", 0, SEED); + }: { + for call in payout_calls { + call.dispatch(RawOrigin::Signed(caller.clone()).into())?; + } + } } #[cfg(test)] mod tests { - use crate::*; - use crate::mock::*; + use super::*; + use crate::mock::{ExtBuilder, Test, Balances, Staking, Origin}; use frame_support::assert_ok; - use crate::benchmarking::{ - create_validators_with_nominators_for_era, - create_validator_with_nominators, - create_nominator_with_validators, - }; - #[test] fn create_validators_with_nominators_for_era_works() { - ExtBuilder::default().stakers(false).build().execute_with(|| { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { let v = 10; let n = 100; @@ -418,47 +390,68 @@ mod tests { #[test] fn create_validator_with_nominators_works() { - ExtBuilder::default().stakers(false).build().execute_with(|| { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { let n = 10; - let validator = create_validator_with_nominators::( + let validator_stash = create_validator_with_nominators::( n, MAX_NOMINATIONS as u32, ).unwrap(); let current_era = CurrentEra::get().unwrap(); - let controller = validator; - let ledger = Staking::ledger(&controller).unwrap(); - let stash = &ledger.stash; - let original_free_balance = Balances::free_balance(stash); - assert_ok!(Staking::payout_validator(Origin::signed(controller), current_era)); - let new_free_balance = Balances::free_balance(stash); + let original_free_balance = Balances::free_balance(&validator_stash); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), validator_stash, current_era)); + let new_free_balance = Balances::free_balance(&validator_stash); assert!(original_free_balance < new_free_balance); }); } #[test] - fn create_nominator_with_validators_works() { - ExtBuilder::default().stakers(false).build().execute_with(|| { - let v = 5; - - let (nominator, validators) = create_nominator_with_validators::(v).unwrap(); - - let current_era = CurrentEra::get().unwrap(); - let controller = nominator; - let ledger = Staking::ledger(&controller).unwrap(); - let stash = &ledger.stash; - - let find_nominator = validators.into_iter().map(|x| (x, 0)).collect(); + fn test_payout_all() { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + let v = 10; + let n = 100; - let original_free_balance = Balances::free_balance(stash); - assert_ok!(Staking::payout_nominator(Origin::signed(controller), current_era, find_nominator)); - let new_free_balance = Balances::free_balance(stash); + let selected_benchmark = SelectedBenchmark::payout_all; + let c = vec![(frame_benchmarking::BenchmarkParameter::v, v), (frame_benchmarking::BenchmarkParameter::n, n)]; + let closure_to_benchmark = + >::instance( + &selected_benchmark, + &c + ).unwrap(); - assert!(original_free_balance < new_free_balance); + assert_ok!(closure_to_benchmark()); }); } + #[test] + fn test_benchmarks() { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + assert_ok!(test_benchmark_bond::()); + assert_ok!(test_benchmark_bond_extra::()); + assert_ok!(test_benchmark_unbond::()); + assert_ok!(test_benchmark_withdraw_unbonded::()); + assert_ok!(test_benchmark_validate::()); + assert_ok!(test_benchmark_nominate::()); + assert_ok!(test_benchmark_chill::()); + assert_ok!(test_benchmark_set_payee::()); + assert_ok!(test_benchmark_set_controller::()); + assert_ok!(test_benchmark_set_validator_count::()); + assert_ok!(test_benchmark_force_no_eras::()); + assert_ok!(test_benchmark_force_new_era::()); + assert_ok!(test_benchmark_force_new_era_always::()); + assert_ok!(test_benchmark_set_invulnerables::()); + assert_ok!(test_benchmark_force_unstake::()); + assert_ok!(test_benchmark_cancel_deferred_slash::()); + assert_ok!(test_benchmark_payout_stakers::()); + assert_ok!(test_benchmark_rebond::()); + assert_ok!(test_benchmark_set_history_depth::()); + assert_ok!(test_benchmark_reap_stash::()); + assert_ok!(test_benchmark_new_era::()); + assert_ok!(test_benchmark_do_slash::()); + assert_ok!(test_benchmark_payout_all::()); + }); + } } diff --git a/frame/staking/src/inflation.rs b/frame/staking/src/inflation.rs index e75ac3af2bf332cc7f357807da33d386732d12f0..d20741d9bc44d656134bfe9297268f41ea3b52b8 100644 --- a/frame/staking/src/inflation.rs +++ b/frame/staking/src/inflation.rs @@ -19,7 +19,7 @@ //! The staking rate in NPoS is the total amount of tokens staked by nominators and validators, //! divided by the total token supply. -use sp_runtime::{Perbill, PerThing, traits::AtLeast32Bit, curve::PiecewiseLinear}; +use sp_runtime::{Perbill, traits::AtLeast32Bit, curve::PiecewiseLinear}; /// The total payout to all validators (and their nominators) per era. /// diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 53ef7b41e43606ad6cb66a46b41f37f5236032cc..86e76408a44f98111ddd56bfe9111540abbd5e67 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -72,8 +72,8 @@ //! There are three possible roles that any staked account pair can be in: `Validator`, `Nominator` //! and `Idle` (defined in [`StakerStatus`](./enum.StakerStatus.html)). There are three //! corresponding instructions to change between roles, namely: -//! [`validate`](./enum.Call.html#variant.validate), [`nominate`](./enum.Call.html#variant.nominate), -//! and [`chill`](./enum.Call.html#variant.chill). +//! [`validate`](./enum.Call.html#variant.validate), +//! [`nominate`](./enum.Call.html#variant.nominate), and [`chill`](./enum.Call.html#variant.chill). //! //! #### Validating //! @@ -104,10 +104,11 @@ //! The **reward and slashing** procedure is the core of the Staking module, attempting to _embrace //! valid behavior_ while _punishing any misbehavior or lack of availability_. //! -//! Reward must be claimed by stakers for each era before it gets too old by $HISTORY_DEPTH using -//! `payout_nominator` and `payout_validator` calls. +//! Rewards must be claimed for each era before it gets too old by `$HISTORY_DEPTH` using the +//! `payout_stakers` call. Any account can call `payout_stakers`, which pays the reward to +//! the validator as well as its nominators. //! Only the [`T::MaxNominatorRewardedPerValidator`] biggest stakers can claim their reward. This -//! limit the i/o cost to compute nominators payout. +//! is to limit the i/o cost to mutate storage for each nominator's account. //! //! Slashing can occur at any point in time, once misbehavior is reported. Once slashing is //! determined, a value is deducted from the balance of the validator and all the nominators who @@ -157,6 +158,7 @@ //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { //! /// Reward a validator. +//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] //! pub fn reward_myself(origin) -> dispatch::DispatchResult { //! let reported = ensure_signed(origin)?; //! >::reward_by_ids(vec![(reported, 10)]); @@ -183,8 +185,8 @@ //! [`reward_by_indices`](./enum.Call.html#variant.reward_by_indices). //! //! [`Module`](./struct.Module.html) implements -//! [`pallet_authorship::EventHandler`](../pallet_authorship/trait.EventHandler.html) to add reward points -//! to block producer and block producer of referenced uncles. +//! [`pallet_authorship::EventHandler`](../pallet_authorship/trait.EventHandler.html) to add reward +//! points to block producer and block producer of referenced uncles. //! //! The validator and its nominator split their reward as following: //! @@ -243,39 +245,55 @@ //! ## Related Modules //! //! - [Balances](../pallet_balances/index.html): Used to manage values at stake. -//! - [Session](../pallet_session/index.html): Used to manage sessions. Also, a list of new validators -//! is stored in the Session module's `Validators` at the end of each era. +//! - [Session](../pallet_session/index.html): Used to manage sessions. Also, a list of new +//! validators is stored in the Session module's `Validators` at the end of each era. -#![recursion_limit="128"] +#![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] #[cfg(test)] mod mock; #[cfg(test)] mod tests; -mod slashing; +#[cfg(feature = "testing-utils")] +pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; +pub mod slashing; +pub mod offchain_election; pub mod inflation; -use sp_std::{prelude::*, result, collections::btree_map::BTreeMap}; +use sp_std::{ + result, + prelude::*, + collections::btree_map::BTreeMap, + convert::{TryInto, From}, + mem::size_of, +}; use codec::{HasCompact, Encode, Decode}; use frame_support::{ - decl_module, decl_event, decl_storage, ensure, decl_error, weights::SimpleDispatchInfo, - dispatch::DispatchResult, storage::IterableStorageMap, traits::{ + decl_module, decl_event, decl_storage, ensure, decl_error, debug, + weights::{SimpleDispatchInfo, Weight}, + storage::IterableStorageMap, + dispatch::{IsSubType, DispatchResult}, + traits::{ Currency, LockIdentifier, LockableCurrency, WithdrawReasons, OnUnbalanced, Imbalance, Get, - Time + UnixTime, EstimateNextNewSession, EnsureOrigin, } }; -use pallet_session::historical::SessionManager; +use pallet_session::historical; use sp_runtime::{ - Perbill, PerThing, RuntimeDebug, + Perbill, PerU16, PerThing, RuntimeDebug, curve::PiecewiseLinear, traits::{ - Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, - AtLeast32Bit, EnsureOrigin, - } + Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, AtLeast32Bit, + SignedExtension, Dispatchable, DispatchInfoOf, + }, + transaction_validity::{ + TransactionValidityError, TransactionValidity, ValidTransaction, InvalidTransaction, + TransactionSource, TransactionPriority, + }, }; use sp_staking::{ SessionIndex, @@ -283,14 +301,33 @@ use sp_staking::{ }; #[cfg(feature = "std")] use sp_runtime::{Serialize, Deserialize}; -use frame_system::{self as system, ensure_signed, ensure_root}; - -use sp_phragmen::ExtendedBalance; +use frame_system::{ + self as system, ensure_signed, ensure_root, ensure_none, + offchain::SubmitUnsignedTransaction, +}; +use sp_phragmen::{ + ExtendedBalance, Assignment, PhragmenScore, PhragmenResult, build_support_map, evaluate_support, + elect, generate_compact_solution_type, is_score_better, VotingLimit, SupportMap, +}; const DEFAULT_MINIMUM_VALIDATOR_COUNT: u32 = 4; -pub const MAX_NOMINATIONS: usize = 16; -const MAX_UNLOCKING_CHUNKS: usize = 32; const STAKING_ID: LockIdentifier = *b"staking "; +pub const MAX_UNLOCKING_CHUNKS: usize = 32; +pub const MAX_NOMINATIONS: usize = ::LIMIT; + +/// Data type used to index nominators in the compact type +pub type NominatorIndex = u32; + +/// Data type used to index validators in the compact type. +pub type ValidatorIndex = u16; + +// Ensure the size of both ValidatorIndex and NominatorIndex. They both need to be well below usize. +static_assertions::const_assert!(size_of::() <= size_of::()); +static_assertions::const_assert!(size_of::() <= size_of::()); + +/// Maximum number of stakers that can be stored in a snapshot. +pub(crate) const MAX_VALIDATORS: usize = ValidatorIndex::max_value() as usize; +pub(crate) const MAX_NOMINATORS: usize = NominatorIndex::max_value() as usize; /// Counter for the number of eras that have passed. pub type EraIndex = u32; @@ -298,18 +335,40 @@ pub type EraIndex = u32; /// Counter for the number of "reward" points earned by a given validator. pub type RewardPoint = u32; +// Note: Maximum nomination limit is set here -- 16. +generate_compact_solution_type!(pub GenericCompactAssignments, 16); + /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] -pub struct ActiveEraInfo { +pub struct ActiveEraInfo { /// Index of era. index: EraIndex, - /// Moment of start + /// Moment of start expresed as millisecond from `$UNIX_EPOCH`. /// /// Start can be none if start hasn't been set for the era yet, /// Start is set on the first on_finalize of the era to guarantee usage of `Time`. - start: Option, + start: Option, } +/// Accuracy used for on-chain phragmen. +pub type ChainAccuracy = Perbill; + +/// Accuracy used for off-chain phragmen. This better be small. +pub type OffchainAccuracy = PerU16; + +/// The balance type of this module. +pub type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; + +/// The compact type for election solutions. +pub type CompactAssignments = + GenericCompactAssignments; + +type PositiveImbalanceOf = + <::Currency as Currency<::AccountId>>::PositiveImbalance; +type NegativeImbalanceOf = + <::Currency as Currency<::AccountId>>::NegativeImbalance; + /// Reward points of an era. Used to split era total payout between validators. /// /// This points will be used to reward validators and their respective nominators. @@ -394,8 +453,9 @@ pub struct StakingLedger { /// Any balance that is becoming free, which may eventually be transferred out /// of the stash (assuming it doesn't get slashed first). pub unlocking: Vec>, - /// The latest and highest era which the staker has claimed reward for. - pub last_reward: Option, + /// List of eras for which the stakers behind a validator have claimed rewards. Only updated + /// for validators. + pub claimed_rewards: Vec, } impl< @@ -420,7 +480,7 @@ impl< total, active: self.active, unlocking, - last_reward: self.last_reward + claimed_rewards: self.claimed_rewards } } @@ -523,10 +583,10 @@ pub struct Nominations { #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, RuntimeDebug)] pub struct IndividualExposure { /// The stash account of the nominator in question. - who: AccountId, + pub who: AccountId, /// Amount of funds exposed. #[codec(compact)] - value: Balance, + pub value: Balance, } /// A snapshot of the stake backing a single validator in the system. @@ -558,13 +618,61 @@ pub struct UnappliedSlash { payout: Balance, } -pub type BalanceOf = - <::Currency as Currency<::AccountId>>::Balance; -type PositiveImbalanceOf = - <::Currency as Currency<::AccountId>>::PositiveImbalance; -type NegativeImbalanceOf = - <::Currency as Currency<::AccountId>>::NegativeImbalance; -type MomentOf = <::Time as Time>::Moment; +/// Indicate how an election round was computed. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug)] +pub enum ElectionCompute { + /// Result was forcefully computed on chain at the end of the session. + OnChain, + /// Result was submitted and accepted to the chain via a signed transaction. + Signed, + /// Result was submitted and accepted to the chain via an unsigned transaction (by an + /// authority). + Unsigned, +} + +/// The result of an election round. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] +pub struct ElectionResult { + /// Flat list of validators who have been elected. + elected_stashes: Vec, + /// Flat list of new exposures, to be updated in the [`Exposure`] storage. + exposures: Vec<(AccountId, Exposure)>, + /// Type of the result. This is kept on chain only to track and report the best score's + /// submission type. An optimisation could remove this. + compute: ElectionCompute, +} + +/// The status of the upcoming (offchain) election. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] +pub enum ElectionStatus { + /// Nothing has and will happen for now. submission window is not open. + Closed, + /// The submission window has been open since the contained block number. + Open(BlockNumber), +} + +impl ElectionStatus { + fn is_open_at(&self, n: BlockNumber) -> bool { + *self == Self::Open(n) + } + + fn is_closed(&self) -> bool { + match self { + Self::Closed => true, + _ => false + } + } + + fn is_open(&self) -> bool { + !self.is_closed() + } +} + +impl Default for ElectionStatus { + fn default() -> Self { + Self::Closed + } +} /// Means for interacting with a specialized version of the `session` trait. /// @@ -590,7 +698,8 @@ impl SessionInterface<::AccountId> for T whe >, T::SessionHandler: pallet_session::SessionHandler<::AccountId>, T::SessionManager: pallet_session::SessionManager<::AccountId>, - T::ValidatorIdOf: Convert<::AccountId, Option<::AccountId>> + T::ValidatorIdOf: + Convert<::AccountId, Option<::AccountId>>, { fn disable_validator(validator: &::AccountId) -> Result { >::disable(validator) @@ -613,7 +722,7 @@ pub trait Trait: frame_system::Trait { /// /// It is guaranteed to start being called from the first `on_finalize`. Thus value at genesis /// is not used. - type Time: Time; + type UnixTime: UnixTime; /// Convert a balance into a number used for election calculation. /// This must fit into a `u64` but is allowed to be sensibly lossy. @@ -654,11 +763,31 @@ pub trait Trait: frame_system::Trait { /// The NPoS reward curve to use. type RewardCurve: Get<&'static PiecewiseLinear<'static>>; - /// The maximum number of nominator rewarded for each validator. + /// Something that can estimate the next session change, accurately or as a best effort guess. + type NextNewSession: EstimateNextNewSession; + + /// How many blocks ahead of the era, within the last do we try to run the phragmen offchain? + /// Setting this to zero will disable the offchain compute and only on-chain seq-phragmen will + /// be used. + type ElectionLookahead: Get; + + /// The overarching call type. + type Call: Dispatchable + From> + IsSubType, Self> + Clone; + + /// A transaction submitter. + type SubmitTransaction: SubmitUnsignedTransaction::Call>; + + /// The maximum number of nominators rewarded for each validator. /// /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim /// their reward. This used to limit the i/o cost for the nominator payout. type MaxNominatorRewardedPerValidator: Get; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; } /// Mode of era-forcing. @@ -686,23 +815,24 @@ impl Default for Forcing { enum Releases { V1_0_0Ancient, V2_0_0, + V3_0_0, } impl Default for Releases { fn default() -> Self { - Releases::V2_0_0 + Releases::V3_0_0 } } decl_storage! { trait Store for Module as Staking { - /// Number of era to keep in history. + /// Number of eras to keep in history. /// - /// Information is kept for eras in `[current_era - history_depth; current_era] + /// Information is kept for eras in `[current_era - history_depth; current_era]`. /// - /// Must be more than the number of era delayed by session otherwise. - /// i.e. active era must always be in history. - /// i.e. `active_era > current_era - history_depth` must be guaranteed. + /// Must be more than the number of eras delayed by session otherwise. + /// I.e. active era must always be in history. + /// I.e. `active_era > current_era - history_depth` must be guaranteed. HistoryDepth get(fn history_depth) config(): u32 = 84; /// The ideal number of staking participants. @@ -738,7 +868,7 @@ decl_storage! { /// The current era index. /// - /// This is the latest planned era, depending on how session module queues the validator + /// This is the latest planned era, depending on how the Session pallet queues the validator /// set, it might be active or not. pub CurrentEra get(fn current_era): Option; @@ -746,9 +876,9 @@ decl_storage! { /// /// The active era is the era currently rewarded. /// Validator set of this era must be equal to `SessionInterface::validators`. - pub ActiveEra get(fn active_era): Option>>; + pub ActiveEra get(fn active_era): Option; - /// The session index at which the era start for the last `HISTORY_DEPTH` eras + /// The session index at which the era start for the last `HISTORY_DEPTH` eras. pub ErasStartSessionIndex get(fn eras_start_session_index): map hasher(twox_64_concat) EraIndex => Option; @@ -764,7 +894,7 @@ decl_storage! { /// Clipped Exposure of validator at era. /// - /// This is similar to [`ErasStakers`] but number of nominators exposed is reduce to the + /// This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the /// `T::MaxNominatorRewardedPerValidator` biggest stakers. /// (Note: the field `total` and `own` of the exposure remains unchanged). /// This is used to limit the i/o cost for the nominator payout. @@ -777,7 +907,7 @@ decl_storage! { double_map hasher(twox_64_concat) EraIndex, hasher(twox_64_concat) T::AccountId => Exposure>; - /// Similarly to `ErasStakers` this holds the preferences of validators. + /// Similar to `ErasStakers`, this holds the preferences of validators. /// /// This is keyed first by the era index to allow bulk deletion and then the stash account. /// @@ -803,7 +933,7 @@ decl_storage! { pub ErasTotalStake get(fn eras_total_stake): map hasher(twox_64_concat) EraIndex => BalanceOf; - /// True if the next session change will be a new era regardless of index. + /// Mode of era forcing. pub ForceEra get(fn force_era) config(): Forcing; /// The percentage of the slash that is distributed to reporters. @@ -848,10 +978,37 @@ decl_storage! { /// The earliest era for which we have a pending, unapplied slash. EarliestUnappliedSlash: Option; + /// Snapshot of validators at the beginning of the current election window. This should only + /// have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`. + pub SnapshotValidators get(fn snapshot_validators): Option>; + + /// Snapshot of nominators at the beginning of the current election window. This should only + /// have a value when [`EraElectionStatus`] == `ElectionStatus::Open(_)`. + pub SnapshotNominators get(fn snapshot_nominators): Option>; + + /// The next validator set. At the end of an era, if this is available (potentially from the + /// result of an offchain worker), it is immediately used. Otherwise, the on-chain election + /// is executed. + pub QueuedElected get(fn queued_elected): Option>>; + + /// The score of the current [`QueuedElected`]. + pub QueuedScore get(fn queued_score): Option; + + /// Flag to control the execution of the offchain election. When `Open(_)`, we accept + /// solutions to be submitted. + pub EraElectionStatus get(fn era_election_status): ElectionStatus; + + /// True if the current planned session is final. + pub IsCurrentSessionFinal get(fn is_current_session_final): bool = false; + + /// True if network has been upgraded to this version. /// Storage version of the pallet. /// - /// This is set to v2.0.0 for new networks. - StorageVersion build(|_: &GenesisConfig| Releases::V2_0_0): Releases; + /// This is set to v3.0.0 for new networks. + StorageVersion build(|_: &GenesisConfig| Releases::V3_0_0): Releases; + + /// The era where we migrated from Lazy Payouts to Simple Payouts + MigrateEra: Option; } add_extra_genesis { config(stakers): @@ -889,13 +1046,15 @@ decl_storage! { decl_event!( pub enum Event where Balance = BalanceOf, ::AccountId { - /// The staker has been rewarded by this amount. AccountId is controller account. + /// The staker has been rewarded by this amount. `AccountId` is the stash account. Reward(AccountId, Balance), /// One validator (and its nominators) has been slashed by the given amount. Slash(AccountId, Balance), /// An old slashing report from a prior era was discarded because it could /// not be processed. OldSlashingReportDiscarded(SessionIndex), + /// A new set of stakers was elected with the given computation method. + StakingElection(ElectionCompute), /// An account has bonded this amount. /// /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, @@ -940,6 +1099,35 @@ decl_error! { InvalidNumberOfNominations, /// Items are not sorted and unique. NotSortedAndUnique, + /// Rewards for this era have already been claimed for this validator. + AlreadyClaimed, + /// The submitted result is received out of the open window. + PhragmenEarlySubmission, + /// The submitted result is not as good as the one stored on chain. + PhragmenWeakSubmission, + /// The snapshot data of the current window is missing. + SnapshotUnavailable, + /// Incorrect number of winners were presented. + PhragmenBogusWinnerCount, + /// One of the submitted winners is not an active candidate on chain (index is out of range + /// in snapshot). + PhragmenBogusWinner, + /// Error while building the assignment type from the compact. This can happen if an index + /// is invalid, or if the weights _overflow_. + PhragmenBogusCompact, + /// One of the submitted nominators is not an active nominator on chain. + PhragmenBogusNominator, + /// One of the submitted nominators has an edge to which they have not voted on chain. + PhragmenBogusNomination, + /// One of the submitted nominators has an edge which is submitted before the last non-zero + /// slash of the target. + PhragmenSlashedNomination, + /// A self vote must only be originated from a validator to ONLY themselves. + PhragmenBogusSelfVote, + /// The submitted result has unknown edges that are not among the presented winners. + PhragmenBogusEdge, + /// The claimed score does not match with the one computed from the data. + PhragmenBogusScore, } } @@ -955,16 +1143,103 @@ decl_module! { fn deposit_event() = default; + /// sets `ElectionStatus` to `Open(now)` where `now` is the block number at which the + /// election window has opened, if we are at the last session and less blocks than + /// `T::ElectionLookahead` is remaining until the next new session schedule. The offchain + /// worker, if applicable, will execute at the end of the current block, and solutions may + /// be submitted. + fn on_initialize(now: T::BlockNumber) -> Weight { + if + // if we don't have any ongoing offchain compute. + Self::era_election_status().is_closed() && + Self::is_current_session_final() + { + if let Some(next_session_change) = T::NextNewSession::estimate_next_new_session(now){ + if let Some(remaining) = next_session_change.checked_sub(&now) { + if remaining <= T::ElectionLookahead::get() && !remaining.is_zero() { + // create snapshot. + if Self::create_stakers_snapshot() { + // Set the flag to make sure we don't waste any compute here in the same era + // after we have triggered the offline compute. + >::put( + ElectionStatus::::Open(now) + ); + debug::native::info!( + target: "staking", + "Election window is Open({:?}). Snapshot created", + now, + ); + } else { + debug::native::warn!( + target: "staking", + "Failed to create snapshot at {:?}. Election window will remain closed.", + now, + ); + } + + } + } + } else { + debug::native::warn!( + target: "staking", + "estimate_next_new_session() failed to execute. Election status cannot be changed.", + ); + } + } + + // weight + 50_000 + } + + /// Check if the current block number is the one at which the election window has been set + /// to open. If so, it runs the offchain worker code. + fn offchain_worker(now: T::BlockNumber) { + use offchain_election::{set_check_offchain_execution_status, compute_offchain_election}; + + if Self::era_election_status().is_open_at(now) { + let offchain_status = set_check_offchain_execution_status::(now); + if let Err(why) = offchain_status { + debug::native::warn!( + target: "staking", + "skipping offchain worker in open election window due to [{}]", + why, + ); + } else { + if let Err(e) = compute_offchain_election::() { + debug::native::warn!( + target: "staking", + "Error in phragmen offchain worker: {:?}", + e, + ); + } else { + debug::native::debug!( + target: "staking", + "Executed offchain worker thread without errors. Transaction submitted to the pool.", + ); + } + } + } + } + fn on_finalize() { // Set the start of the first era. if let Some(mut active_era) = Self::active_era() { if active_era.start.is_none() { - active_era.start = Some(T::Time::now()); - >::put(active_era); + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); + active_era.start = Some(now_as_millis_u64); + ActiveEra::put(active_era); } } } + fn on_runtime_upgrade() -> Weight { + // For Kusama the type hasn't actually changed as Moment was u64 and was the number of + // millisecond since unix epoch. + StorageVersion::put(Releases::V3_0_0); + Self::migrate_last_reward_to_claimed_rewards(); + 0 + } + /// Take the origin account as a stash and lock up `value` of its balance. `controller` will /// be the account that controls it. /// @@ -979,14 +1254,14 @@ decl_module! { /// - O(1). /// - Three extra DB entries. /// - /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned unless - /// the `origin` falls below _existential deposit_ and gets removed as dust. + /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned + /// unless the `origin` falls below _existential deposit_ and gets removed as dust. /// # #[weight = SimpleDispatchInfo::FixedNormal(500_000)] - fn bond(origin, + pub fn bond(origin, controller: ::Source, #[compact] value: BalanceOf, - payee: RewardDestination + payee: RewardDestination, ) { let stash = ensure_signed(origin)?; @@ -1012,6 +1287,10 @@ decl_module! { system::Module::::inc_ref(&stash); + let current_era = CurrentEra::get().unwrap_or(0); + let history_depth = Self::history_depth(); + let last_reward_era = current_era.saturating_sub(history_depth); + let stash_balance = T::Currency::free_balance(&stash); let value = value.min(stash_balance); Self::deposit_event(RawEvent::Bonded(stash.clone(), value)); @@ -1020,7 +1299,7 @@ decl_module! { total: value, active: value, unlocking: vec![], - last_reward: Self::current_era(), + claimed_rewards: (last_reward_era..current_era).collect(), }; Self::update_ledger(&controller, &item); } @@ -1173,7 +1452,7 @@ decl_module! { /// - Writes are limited to the `origin` account key. /// # #[weight = SimpleDispatchInfo::FixedNormal(750_000)] - fn validate(origin, prefs: ValidatorPrefs) { + pub fn validate(origin, prefs: ValidatorPrefs) { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; let stash = &ledger.stash; @@ -1189,17 +1468,17 @@ decl_module! { /// /// # /// - The transaction's complexity is proportional to the size of `targets`, - /// which is capped at `MAX_NOMINATIONS`. + /// which is capped at CompactAssignments::LIMIT. /// - Both the reads and writes follow a similar pattern. /// # #[weight = SimpleDispatchInfo::FixedNormal(750_000)] - fn nominate(origin, targets: Vec<::Source>) { + pub fn nominate(origin, targets: Vec<::Source>) { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; let stash = &ledger.stash; ensure!(!targets.is_empty(), Error::::EmptyTargets); let targets = targets.into_iter() - .take(MAX_NOMINATIONS) + .take(::LIMIT) .map(|t| T::Lookup::lookup(t)) .collect::, _>>()?; @@ -1368,6 +1647,10 @@ decl_module! { ::UnappliedSlashes::insert(&era, &unapplied); } + /// **This extrinsic will be removed after `MigrationEra + HistoryDepth` has passed, giving + /// opportunity for users to claim all rewards before moving to Simple Payouts. After this + /// time, you should use `payout_stakers` instead.** + /// /// Make one nominator's payout for one era. /// /// - `who` is the controller account of the nominator to pay out. @@ -1402,6 +1685,10 @@ decl_module! { Self::do_payout_nominator(who, era, validators) } + /// **This extrinsic will be removed after `MigrationEra + HistoryDepth` has passed, giving + /// opportunity for users to claim all rewards before moving to Simple Payouts. After this + /// time, you should use `payout_stakers` instead.** + /// /// Make one validator's payout for one era. /// /// - `who` is the controller account of the validator to pay out. @@ -1423,6 +1710,25 @@ decl_module! { Self::do_payout_validator(who, era) } + /// Pay out all the stakers behind a single validator for a single era. + /// + /// - `validator_stash` is the stash account of the validator. Their nominators, up to + /// `T::MaxNominatorRewardedPerValidator`, will also receive their rewards. + /// - `era` may be any era between `[current_era - history_depth; current_era]`. + /// + /// The origin of this call must be _Signed_. Any account can call this function, even if + /// it is not one of the stakers. + /// + /// # + /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). + /// - Contains a limited number of reads and writes. + /// # + #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + fn payout_stakers(origin, validator_stash: T::AccountId, era: EraIndex) -> DispatchResult { + ensure_signed(origin)?; + Self::do_payout_stakers(validator_stash, era) + } + /// Rebond a portion of the stash scheduled to be unlocked. /// /// # @@ -1467,23 +1773,222 @@ decl_module! { /// This can be called from any origin. /// /// - `stash`: The stash account to reap. Its balance must be zero. + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn reap_stash(_origin, stash: T::AccountId) { ensure!(T::Currency::total_balance(&stash).is_zero(), Error::::FundedTarget); Self::kill_stash(&stash)?; T::Currency::remove_lock(STAKING_ID, &stash); } + + /// Submit a phragmen result to the chain. If the solution: + /// + /// 1. is valid. + /// 2. has a better score than a potentially existing solution on chain. + /// + /// then, it will be _put_ on chain. + /// + /// A solution consists of two pieces of data: + /// + /// 1. `winners`: a flat vector of all the winners of the round. + /// 2. `assignments`: the compact version of an assignment vector that encodes the edge + /// weights. + /// + /// Both of which may be computed using [`phragmen`], or any other algorithm. + /// + /// Additionally, the submitter must provide: + /// + /// - The `score` that they claim their solution has. + /// + /// Both validators and nominators will be represented by indices in the solution. The + /// indices should respect the corresponding types ([`ValidatorIndex`] and + /// [`NominatorIndex`]). Moreover, they should be valid when used to index into + /// [`SnapshotValidators`] and [`SnapshotNominators`]. Any invalid index will cause the + /// solution to be rejected. These two storage items are set during the election window and + /// may be used to determine the indices. + /// + /// A solution is valid if: + /// + /// 0. It is submitted when [`EraElectionStatus`] is `Open`. + /// 1. Its claimed score is equal to the score computed on-chain. + /// 2. Presents the correct number of winners. + /// 3. All indexes must be value according to the snapshot vectors. All edge values must + /// also be correct and should not overflow the granularity of the ratio type (i.e. 256 + /// or billion). + /// 4. For each edge, all targets are actually nominated by the voter. + /// 5. Has correct self-votes. + /// + /// A solutions score is consisted of 3 parameters: + /// + /// 1. `min { support.total }` for each support of a winner. This value should be maximized. + /// 2. `sum { support.total }` for each support of a winner. This value should be minimized. + /// 3. `sum { support.total^2 }` for each support of a winner. This value should be + /// minimized (to ensure less variance) + /// + /// # + /// E: number of edges. m: size of winner committee. n: number of nominators. d: edge degree + /// (16 for now) v: number of on-chain validator candidates. + /// + /// NOTE: given a solution which is reduced, we can enable a new check the ensure `|E| < n + + /// m`. We don't do this _yet_, but our offchain worker code executes it nonetheless. + /// + /// major steps (all done in `check_and_replace_solution`): + /// + /// - Storage: O(1) read `ElectionStatus`. + /// - Storage: O(1) read `PhragmenScore`. + /// - Storage: O(1) read `ValidatorCount`. + /// - Storage: O(1) length read from `SnapshotValidators`. + /// + /// - Storage: O(v) reads of `AccountId` to fetch `snapshot_validators`. + /// - Memory: O(m) iterations to map winner index to validator id. + /// - Storage: O(n) reads `AccountId` to fetch `snapshot_nominators`. + /// - Memory: O(n + m) reads to map index to `AccountId` for un-compact. + /// + /// - Storage: O(e) accountid reads from `Nomination` to read correct nominations. + /// - Storage: O(e) calls into `slashable_balance_of_extended` to convert ratio to staked. + /// + /// - Memory: build_support_map. O(e). + /// - Memory: evaluate_support: O(E). + /// + /// - Storage: O(e) writes to `QueuedElected`. + /// - Storage: O(1) write to `QueuedScore` + /// + /// The weight of this call is 1/10th of the blocks total weight. + /// # + #[weight = SimpleDispatchInfo::FixedNormal(100_000_000)] + pub fn submit_election_solution( + origin, + winners: Vec, + compact_assignments: CompactAssignments, + score: PhragmenScore, + era: EraIndex, + ) { + let _who = ensure_signed(origin)?; + Self::check_and_replace_solution( + winners, + compact_assignments, + ElectionCompute::Signed, + score, + era, + )? + } + + /// Unsigned version of `submit_election_solution`. + /// + /// Note that this must pass the [`ValidateUnsigned`] check which only allows transactions + /// from the local node to be included. In other words, only the block author can include a + /// transaction in the block. + #[weight = SimpleDispatchInfo::FixedNormal(100_000_000)] + pub fn submit_election_solution_unsigned( + origin, + winners: Vec, + compact_assignments: CompactAssignments, + score: PhragmenScore, + era: EraIndex, + ) { + ensure_none(origin)?; + Self::check_and_replace_solution( + winners, + compact_assignments, + ElectionCompute::Unsigned, + score, + era, + )? + // TODO: instead of returning an error, panic. This makes the entire produced block + // invalid. + // This ensures that block authors will not ever try and submit a solution which is not + // an improvement, since they will lose their authoring points/rewards. + } } } impl Module { - // PUBLIC IMMUTABLES + /// Migrate `last_reward` to `claimed_rewards` + pub fn migrate_last_reward_to_claimed_rewards() { + use frame_support::migration::{StorageIterator, put_storage_value}; + // Migrate from `last_reward` to `claimed_rewards`. + // We will construct a vector from `current_era - history_depth` to `last_reward` + // for each validator and nominator. + // + // Old Staking Ledger + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] + struct OldStakingLedger { + pub stash: AccountId, + #[codec(compact)] + pub total: Balance, + #[codec(compact)] + pub active: Balance, + pub unlocking: Vec>, + pub last_reward: Option, + } + // Current era and history depth + let current_era = Self::current_era().unwrap_or(0); + let history_depth = Self::history_depth(); + let last_payout_era = current_era.saturating_sub(history_depth); + // Convert all ledgers to the new format. + for (hash, old_ledger) in StorageIterator::>>::new(b"Staking", b"Ledger").drain() { + let last_reward = old_ledger.last_reward.unwrap_or(0); + let new_ledger = StakingLedger { + stash: old_ledger.stash, + total: old_ledger.total, + active: old_ledger.active, + unlocking: old_ledger.unlocking, + claimed_rewards: (last_payout_era..=last_reward).collect(), + }; + put_storage_value(b"Staking", b"Ledger", &hash, new_ledger); + } + MigrateEra::put(current_era); + } /// The total balance that can be slashed from a stash account as of right now. pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() } - // MUTABLES (DANGEROUS) + /// internal impl of [`slashable_balance_of`] that returns [`ExtendedBalance`]. + fn slashable_balance_of_extended(stash: &T::AccountId) -> ExtendedBalance { + , u64>>::convert( + Self::slashable_balance_of(stash) + ) as ExtendedBalance + } + + /// Dump the list of validators and nominators into vectors and keep them on-chain. + /// + /// This data is used to efficiently evaluate election results. returns `true` if the operation + /// is successful. + fn create_stakers_snapshot() -> bool { + let validators = >::iter().map(|(v, _)| v).collect::>(); + let mut nominators = >::iter().map(|(n, _)| n).collect::>(); + + let num_validators = validators.len(); + let num_nominators = nominators.len(); + if + num_validators > MAX_VALIDATORS || + num_nominators.saturating_add(num_validators) > MAX_NOMINATORS + { + debug::native::warn!( + target: "staking", + "Snapshot size too big [{} <> {}][{} <> {}].", + num_validators, + MAX_VALIDATORS, + num_nominators, + MAX_NOMINATORS, + ); + false + } else { + // all validators nominate themselves; + nominators.extend(validators.clone()); + + >::put(validators); + >::put(nominators); + true + } + } + + /// Clears both snapshots of stakers. + fn kill_stakers_snapshot() { + >::kill(); + >::kill(); + } fn do_payout_nominator(who: T::AccountId, era: EraIndex, validators: Vec<(T::AccountId, u32)>) -> DispatchResult @@ -1493,6 +1998,15 @@ impl Module { if validators.len() > MAX_NOMINATIONS { return Err(Error::::InvalidNumberOfNominations.into()); } + // If migrate_era is not populated, then you should use `payout_stakers` + let migrate_era = MigrateEra::get().ok_or(Error::::InvalidEraToReward)?; + // This payout mechanism will only work for eras before the migration. + // Subsequent payouts should use `payout_stakers`. + ensure!(era < migrate_era, Error::::InvalidEraToReward); + let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; + ensure!(era <= current_era, Error::::InvalidEraToReward); + let history_depth = Self::history_depth(); + ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); // Note: if era has no reward to be claimed, era may be future. better not to update // `nominator_ledger.last_reward` in this case. @@ -1501,11 +2015,12 @@ impl Module { let mut nominator_ledger = >::get(&who).ok_or_else(|| Error::::NotController)?; - if nominator_ledger.last_reward.map(|last_reward| last_reward >= era).unwrap_or(false) { - return Err(Error::::InvalidEraToReward.into()); + nominator_ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); + match nominator_ledger.claimed_rewards.binary_search(&era) { + Ok(_) => Err(Error::::AlreadyClaimed)?, + Err(pos) => nominator_ledger.claimed_rewards.insert(pos, era), } - nominator_ledger.last_reward = Some(era); >::insert(&who, &nominator_ledger); let mut reward = Perbill::zero(); @@ -1549,17 +2064,29 @@ impl Module { } fn do_payout_validator(who: T::AccountId, era: EraIndex) -> DispatchResult { + // If migrate_era is not populated, then you should use `payout_stakers` + let migrate_era = MigrateEra::get().ok_or(Error::::InvalidEraToReward)?; + // This payout mechanism will only work for eras before the migration. + // Subsequent payouts should use `payout_stakers`. + ensure!(era < migrate_era, Error::::InvalidEraToReward); + let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; + ensure!(era <= current_era, Error::::InvalidEraToReward); + let history_depth = Self::history_depth(); + ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); + // Note: if era has no reward to be claimed, era may be future. better not to update // `ledger.last_reward` in this case. let era_payout = >::get(&era) .ok_or_else(|| Error::::InvalidEraToReward)?; let mut ledger = >::get(&who).ok_or_else(|| Error::::NotController)?; - if ledger.last_reward.map(|last_reward| last_reward >= era).unwrap_or(false) { - return Err(Error::::InvalidEraToReward.into()); + + ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); + match ledger.claimed_rewards.binary_search(&era) { + Ok(_) => Err(Error::::AlreadyClaimed)?, + Err(pos) => ledger.claimed_rewards.insert(pos, era), } - ledger.last_reward = Some(era); >::insert(&who, &ledger); let era_reward_points = >::get(&era); @@ -1590,6 +2117,108 @@ impl Module { Ok(()) } + fn do_payout_stakers( + validator_stash: T::AccountId, + era: EraIndex, + ) -> DispatchResult { + /* Validate input data */ + let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; + ensure!(era <= current_era, Error::::InvalidEraToReward); + let history_depth = Self::history_depth(); + ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); + + // If there was no migration, then this function is always valid. + if let Some(migrate_era) = MigrateEra::get() { + // This payout mechanism will only work for eras on and after the migration. + // Payouts before then should use `payout_nominator`/`payout_validator`. + ensure!(migrate_era <= era, Error::::InvalidEraToReward); + } + + // Note: if era has no reward to be claimed, era may be future. better not to update + // `ledger.claimed_rewards` in this case. + let era_payout = >::get(&era) + .ok_or_else(|| Error::::InvalidEraToReward)?; + + let controller = Self::bonded(&validator_stash).ok_or(Error::::NotStash)?; + let mut ledger = >::get(&controller).ok_or_else(|| Error::::NotController)?; + + ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); + match ledger.claimed_rewards.binary_search(&era) { + Ok(_) => Err(Error::::AlreadyClaimed)?, + Err(pos) => ledger.claimed_rewards.insert(pos, era), + } + + let exposure = >::get(&era, &ledger.stash); + + /* Input data seems good, no errors allowed after this point */ + + >::insert(&controller, &ledger); + + // Get Era reward points. It has TOTAL and INDIVIDUAL + // Find the fraction of the era reward that belongs to the validator + // Take that fraction of the eras rewards to split to nominator and validator + // + // Then look at the validator, figure out the proportion of their reward + // which goes to them and each of their nominators. + + let era_reward_points = >::get(&era); + let total_reward_points = era_reward_points.total; + let validator_reward_points = era_reward_points.individual.get(&ledger.stash) + .map(|points| *points) + .unwrap_or_else(|| Zero::zero()); + + // Nothing to do if they have no reward points. + if validator_reward_points.is_zero() { return Ok(())} + + // This is the fraction of the total reward that the validator and the + // nominators will get. + let validator_total_reward_part = Perbill::from_rational_approximation( + validator_reward_points, + total_reward_points, + ); + + // This is how much validator + nominators are entitled to. + let validator_total_payout = validator_total_reward_part * era_payout; + + let validator_prefs = Self::eras_validator_prefs(&era, &validator_stash); + // Validator first gets a cut off the top. + let validator_commission = validator_prefs.commission; + let validator_commission_payout = validator_commission * validator_total_payout; + + let validator_leftover_payout = validator_total_payout - validator_commission_payout; + // Now let's calculate how this is split to the validator. + let validator_exposure_part = Perbill::from_rational_approximation( + exposure.own, + exposure.total, + ); + let validator_staking_payout = validator_exposure_part * validator_leftover_payout; + + // We can now make total validator payout: + if let Some(imbalance) = Self::make_payout( + &ledger.stash, + validator_staking_payout + validator_commission_payout + ) { + Self::deposit_event(RawEvent::Reward(ledger.stash, imbalance.peek())); + } + + // Lets now calculate how this is split to the nominators. + // Sort nominators by highest to lowest exposure, but only keep `max_nominator_payouts` of them. + for nominator in exposure.others.iter() { + let nominator_exposure_part = Perbill::from_rational_approximation( + nominator.value, + exposure.total, + ); + + let nominator_reward: BalanceOf = nominator_exposure_part * validator_leftover_payout; + // We can now make nominator payout: + if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { + Self::deposit_event(RawEvent::Reward(nominator.who.clone(), imbalance.peek())); + } + } + + Ok(()) + } + /// Update the ledger for a controller. This will also update the stash lock. The lock will /// will lock the entire funds except paying for further transactions. fn update_ledger( @@ -1652,9 +2281,17 @@ impl Module { Forcing::ForceNew => ForceEra::kill(), Forcing::ForceAlways => (), Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), - _ => return None, + _ => { + // not forcing, not a new era either. If final, set the flag. + if era_length + 1 >= T::SessionsPerEra::get() { + IsCurrentSessionFinal::put(true); + } + return None + }, } + // new era. + IsCurrentSessionFinal::put(false); Self::new_era(session_index) } else { // Set initial era @@ -1662,6 +2299,188 @@ impl Module { } } + /// Basic and cheap checks that we perform in validate unsigned, and in the execution. + pub fn pre_dispatch_checks(score: PhragmenScore, era: EraIndex) -> Result<(), Error> { + // discard solutions that are not in-time + // check window open + ensure!( + Self::era_election_status().is_open(), + Error::::PhragmenEarlySubmission, + ); + + // check current era. + if let Some(current_era) = Self::active_era().map(|e| e.index) { + ensure!( + current_era == era, + Error::::PhragmenEarlySubmission, + ) + } + + // assume the given score is valid. Is it better than what we have on-chain, if we have any? + if let Some(queued_score) = Self::queued_score() { + ensure!( + is_score_better(queued_score, score), + Error::::PhragmenWeakSubmission, + ) + } + + Ok(()) + } + + /// Checks a given solution and if correct and improved, writes it on chain as the queued result + /// of the next round. This may be called by both a signed and an unsigned transaction. + pub fn check_and_replace_solution( + winners: Vec, + compact_assignments: CompactAssignments, + compute: ElectionCompute, + claimed_score: PhragmenScore, + era: EraIndex, + ) -> Result<(), Error> { + // Do the basic checks. era, claimed score and window open. + Self::pre_dispatch_checks(claimed_score, era)?; + + // Check that the number of presented winners is sane. Most often we have more candidates + // that we need. Then it should be Self::validator_count(). Else it should be all the + // candidates. + let snapshot_length = >::decode_len() + .map_err(|_| Error::::SnapshotUnavailable)?; + let desired_winners = Self::validator_count().min(snapshot_length as u32); + ensure!(winners.len() as u32 == desired_winners, Error::::PhragmenBogusWinnerCount); + + // decode snapshot validators. + let snapshot_validators = Self::snapshot_validators() + .ok_or(Error::::SnapshotUnavailable)?; + + // check if all winners were legit; this is rather cheap. Replace with accountId. + let winners = winners.into_iter().map(|widx| { + // NOTE: at the moment, since staking is explicitly blocking any offence until election + // is closed, we don't check here if the account id at `snapshot_validators[widx]` is + // actually a validator. If this ever changes, this loop needs to also check this. + snapshot_validators.get(widx as usize).cloned().ok_or(Error::::PhragmenBogusWinner) + }).collect::, Error>>()?; + + // decode the rest of the snapshot. + let snapshot_nominators = >::snapshot_nominators() + .ok_or(Error::::SnapshotUnavailable)?; + + // helpers + let nominator_at = |i: NominatorIndex| -> Option { + snapshot_nominators.get(i as usize).cloned() + }; + let validator_at = |i: ValidatorIndex| -> Option { + snapshot_validators.get(i as usize).cloned() + }; + + // un-compact. + let assignments = compact_assignments.into_assignment( + nominator_at, + validator_at, + ).map_err(|e| { + // log the error since it is not propagated into the runtime error. + debug::native::warn!( + target: "staking", + "un-compacting solution failed due to {:?}", + e, + ); + Error::::PhragmenBogusCompact + })?; + + // check all nominators actually including the claimed vote. Also check correct self votes. + // Note that we assume all validators and nominators in `assignments` are properly bonded, + // because they are coming from the snapshot via a given index. + for Assignment { who, distribution } in assignments.iter() { + let is_validator = >::contains_key(&who); + let maybe_nomination = Self::nominators(&who); + + if !(maybe_nomination.is_some() ^ is_validator) { + // all of the indices must map to either a validator or a nominator. If this is ever + // not the case, then the locking system of staking is most likely faulty, or we + // have bigger problems. + debug::native::error!( + target: "staking", + "detected an error in the staking locking and snapshot." + ); + // abort. + return Err(Error::::PhragmenBogusNominator); + } + + if !is_validator { + // a normal vote + let nomination = maybe_nomination.expect( + "exactly one of `maybe_validator` and `maybe_nomination.is_some` is true. \ + is_validator is false; maybe_nomination is some; qed" + ); + + // NOTE: we don't really have to check here if the sum of all edges are the + // nominator correct. Un-compacting assures this by definition. + + for (t, _) in distribution { + // each target in the provided distribution must be actually nominated by the + // nominator after the last non-zero slash. + if nomination.targets.iter().find(|&tt| tt == t).is_none() { + return Err(Error::::PhragmenBogusNomination); + } + + if ::SlashingSpans::get(&t).map_or( + false, + |spans| nomination.submitted_in < spans.last_nonzero_slash(), + ) { + return Err(Error::::PhragmenSlashedNomination); + } + } + } else { + // a self vote + ensure!(distribution.len() == 1, Error::::PhragmenBogusSelfVote); + ensure!(distribution[0].0 == *who, Error::::PhragmenBogusSelfVote); + // defensive only. A compact assignment of length one does NOT encode the weight and + // it is always created to be 100%. + ensure!( + distribution[0].1 == OffchainAccuracy::one(), + Error::::PhragmenBogusSelfVote, + ); + } + } + + // convert into staked assignments. + let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + assignments, + Self::slashable_balance_of_extended, + ); + + // build the support map thereof in order to evaluate. + // OPTIMIZATION: loop to create the staked assignments but it would bloat the code. Okay for + // now as it does not add to the complexity order. + let (supports, num_error) = build_support_map::( + &winners, + &staked_assignments, + ); + // This technically checks that all targets in all nominators were among the winners. + ensure!(num_error == 0, Error::::PhragmenBogusEdge); + + // Check if the score is the same as the claimed one. + let submitted_score = evaluate_support(&supports); + ensure!(submitted_score == claimed_score, Error::::PhragmenBogusScore); + + // At last, alles Ok. Exposures and store the result. + let exposures = Self::collect_exposure(supports); + debug::native::info!( + target: "staking", + "A better solution (with compute {:?}) has been validated and stored on chain.", + compute, + ); + + // write new results. + >::put(ElectionResult { + elected_stashes: winners, + compute, + exposures, + }); + QueuedScore::put(submitted_score); + + Ok(()) + + } + /// Start a session potentially starting an era. fn start_session(start_session: SessionIndex) { let next_active_era = Self::active_era().map(|e| e.index + 1).unwrap_or(0); @@ -1696,7 +2515,7 @@ impl Module { /// * reset `active_era.start`, /// * update `BondedEras` and apply slashes. fn start_era(start_session: SessionIndex) { - let active_era = >::mutate(|active_era| { + let active_era = ActiveEra::mutate(|active_era| { let new_index = active_era.as_ref().map(|info| info.index + 1).unwrap_or(0); *active_era = Some(ActiveEraInfo { index: new_index, @@ -1734,12 +2553,12 @@ impl Module { } /// Compute payout for era. - fn end_era(active_era: ActiveEraInfo>, _session_index: SessionIndex) { + fn end_era(active_era: ActiveEraInfo, _session_index: SessionIndex) { // Note: active_era_start can be None if end era is called during genesis config. if let Some(active_era_start) = active_era.start { - let now = T::Time::now(); + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - let era_duration = now - active_era_start; + let era_duration = now_as_millis_u64 - active_era_start; let (total_payout, _max_payout) = inflation::compute_total_payout( &T::RewardCurve::get(), Self::eras_total_stake(&active_era.index), @@ -1768,55 +2587,160 @@ impl Module { } // Set staking information for new era. - let maybe_new_validators = Self::select_validators(current_era); + let maybe_new_validators = Self::select_and_update_validators(current_era); maybe_new_validators } - /// Clear all era information for given era. - fn clear_era_information(era_index: EraIndex) { - >::remove_prefix(era_index); - >::remove_prefix(era_index); - >::remove_prefix(era_index); - >::remove(era_index); - >::remove(era_index); - >::remove(era_index); - ErasStartSessionIndex::remove(era_index); - } + /// Select the new validator set at the end of the era. + /// + /// Runs [`try_do_phragmen`] and updates the following storage items: + /// - [`EraElectionStatus`]: with `None`. + /// - [`ErasStakers`]: with the new staker set. + /// - [`ErasStakersClipped`]. + /// - [`ErasValidatorPrefs`]. + /// - [`ErasTotalStake`]: with the new total stake. + /// - [`SnapshotValidators`] and [`SnapshotNominators`] are both removed. + /// + /// Internally, [`QueuedElected`], snapshots and [`QueuedScore`] are also consumed. + /// + /// If the election has been successful, It passes the new set upwards. + /// + /// This should only be called at the end of an era. + fn select_and_update_validators(current_era: EraIndex) -> Option> { + if let Some(ElectionResult::> { + elected_stashes, + exposures, + compute, + }) = Self::try_do_phragmen() { + // We have chosen the new validator set. Submission is no longer allowed. + >::put(ElectionStatus::Closed); + + // kill the snapshots. + Self::kill_stakers_snapshot(); + + // Populate Stakers and write slot stake. + let mut total_stake: BalanceOf = Zero::zero(); + exposures.into_iter().for_each(|(stash, exposure)| { + total_stake = total_stake.saturating_add(exposure.total); + >::insert(current_era, &stash, &exposure); - /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. - fn apply_unapplied_slashes(active_era: EraIndex) { - let slash_defer_duration = T::SlashDeferDuration::get(); - ::EarliestUnappliedSlash::mutate(|earliest| if let Some(ref mut earliest) = earliest { - let keep_from = active_era.saturating_sub(slash_defer_duration); - for era in (*earliest)..keep_from { - let era_slashes = ::UnappliedSlashes::take(&era); - for slash in era_slashes { - slashing::apply_slash::(slash); + let mut exposure_clipped = exposure; + let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; + if exposure_clipped.others.len() > clipped_max_len { + exposure_clipped.others.sort_unstable_by(|a, b| a.value.cmp(&b.value).reverse()); + exposure_clipped.others.truncate(clipped_max_len); } + >::insert(¤t_era, &stash, exposure_clipped); + }); + + // Insert current era staking information + >::insert(¤t_era, total_stake); + + // collect the pref of all winners + for stash in &elected_stashes { + let pref = Self::validators(stash); + >::insert(¤t_era, stash, pref); } - *earliest = (*earliest).max(keep_from) - }) + // emit event + Self::deposit_event(RawEvent::StakingElection(compute)); + + debug::native::info!( + target: "staking", + "new validator set of size {:?} has been elected via {:?} for era {:?}", + elected_stashes.len(), + compute, + current_era, + ); + + Some(elected_stashes) + } else { + None + } } - /// Select a new validator set from the assembled stakers and their role preferences, and store - /// staking information for the new current era. + /// Select a new validator set from the assembled stakers and their role preferences. It tries + /// first to peek into [`QueuedElected`]. Otherwise, it runs a new phragmen. /// - /// Fill the storages `ErasStakers`, `ErasStakersClipped`, `ErasValidatorPrefs` and - /// `ErasTotalStake` for current era. + /// If [`QueuedElected`] and [`QueuedScore`] exists, they are both removed. No further storage + /// is updated. + fn try_do_phragmen() -> Option>> { + // a phragmen result from either a stored submission or locally executed one. + let next_result = >::take().or_else(|| + Self::do_phragmen_with_post_processing::(ElectionCompute::OnChain) + ); + + // either way, kill this. We remove it here to make sure it always has the exact same + // lifetime as `QueuedElected`. + QueuedScore::kill(); + + next_result + } + + /// Execute phragmen and return the new results. The edge weights are processed into support + /// values. /// - /// Returns a set of newly selected _stash_ IDs. + /// This is basically a wrapper around [`do_phragmen`] which translates `PhragmenResult` into + /// `ElectionResult`. /// - /// Assumes storage is coherent with the declaration. - fn select_validators(current_era: EraIndex) -> Option> { + /// No storage item is updated. + fn do_phragmen_with_post_processing(compute: ElectionCompute) + -> Option>> + where + Accuracy: sp_std::ops::Mul, + ExtendedBalance: From<::Inner>, + { + if let Some(phragmen_result) = Self::do_phragmen::() { + let elected_stashes = phragmen_result.winners.iter() + .map(|(s, _)| s.clone()) + .collect::>(); + let assignments = phragmen_result.assignments; + + let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + assignments, + Self::slashable_balance_of_extended, + ); + + let (supports, _) = build_support_map::( + &elected_stashes, + &staked_assignments, + ); + + // collect exposures + let exposures = Self::collect_exposure(supports); + + // In order to keep the property required by `on_session_ending` that we must return the + // new validator set even if it's the same as the old, as long as any underlying + // economic conditions have changed, we don't attempt to do any optimization where we + // compare against the prior set. + Some(ElectionResult::> { + elected_stashes, + exposures, + compute, + }) + } else { + // There were not enough candidates for even our minimal level of functionality. This is + // bad. We should probably disable all functionality except for block production and let + // the chain keep producing blocks until we can decide on a sufficiently substantial + // set. TODO: #2494 + None + } + } + + /// Execute phragmen and return the new results. No post-processing is applied and the raw edge + /// weights are returned. + /// + /// Self votes are added and nominations before the most recent slashing span are reaped. + /// + /// No storage item is updated. + fn do_phragmen() -> Option> { let mut all_nominators: Vec<(T::AccountId, BalanceOf, Vec)> = Vec::new(); - let mut all_validators_and_prefs = BTreeMap::new(); let mut all_validators = Vec::new(); - for (validator, preference) in >::iter() { + for (validator, _) in >::iter() { + // append self vote let self_vote = (validator.clone(), Self::slashable_balance_of(&validator), vec![validator.clone()]); all_nominators.push(self_vote); - all_validators_and_prefs.insert(validator.clone(), preference); all_validators.push(validator); } @@ -1824,7 +2748,7 @@ impl Module { let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; // Filter out nomination targets which were nominated before the most recent - // non-zero slash. + // slashing span. targets.retain(|stash| { ::SlashingSpans::get(&stash).map_or( true, @@ -1839,92 +2763,44 @@ impl Module { (n, s, ns) })); - let maybe_phragmen_result = sp_phragmen::elect::<_, _, T::CurrencyToVote, Perbill>( + elect::<_, _, T::CurrencyToVote, Accuracy>( Self::validator_count() as usize, Self::minimum_validator_count().max(1) as usize, all_validators, all_nominators, - ); - - if let Some(phragmen_result) = maybe_phragmen_result { - let elected_stashes = phragmen_result.winners.into_iter() - .map(|(s, _)| s) - .collect::>(); - let assignments = phragmen_result.assignments; - - let to_balance = |e: ExtendedBalance| - >>::convert(e); - - let supports = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>( - &elected_stashes, - &assignments, - Self::slashable_balance_of, - ); - - // Populate stakers information and figure out the total stake. - let mut total_staked = BalanceOf::::zero(); - for (c, s) in supports.into_iter() { - // build `struct exposure` from `support` - let mut others = Vec::new(); - let mut own: BalanceOf = Zero::zero(); - let mut total: BalanceOf = Zero::zero(); - s.voters - .into_iter() - .map(|(who, value)| (who, to_balance(value))) - .for_each(|(who, value)| { - if who == c { - own = own.saturating_add(value); - } else { - others.push(IndividualExposure { who, value }); - } - total = total.saturating_add(value); - }); - - total_staked = total_staked.saturating_add(total); - - let exposure = Exposure { - own, - others, - // This might reasonably saturate and we cannot do much about it. The sum of - // someone's stake might exceed the balance type if they have the maximum amount - // of balance and receive some support. This is super unlikely to happen, yet - // we simulate it in some tests. - total, - }; - >::insert(¤t_era, &c, &exposure); + ) + } - let mut exposure_clipped = exposure; - let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; - if exposure_clipped.others.len() > clipped_max_len { - exposure_clipped.others.sort_unstable_by(|a, b| a.value.cmp(&b.value).reverse()); - exposure_clipped.others.truncate(clipped_max_len); - } - >::insert(¤t_era, &c, exposure_clipped); - } + /// Consume a set of [`Supports`] from [`sp_phragmen`] and collect them into a [`Exposure`] + fn collect_exposure(supports: SupportMap) -> Vec<(T::AccountId, Exposure>)> { + let to_balance = |e: ExtendedBalance| + >>::convert(e); + + supports.into_iter().map(|(validator, support)| { + // build `struct exposure` from `support` + let mut others = Vec::new(); + let mut own: BalanceOf = Zero::zero(); + let mut total: BalanceOf = Zero::zero(); + support.voters + .into_iter() + .map(|(nominator, weight)| (nominator, to_balance(weight))) + .for_each(|(nominator, stake)| { + if nominator == validator { + own = own.saturating_add(stake); + } else { + others.push(IndividualExposure { who: nominator, value: stake }); + } + total = total.saturating_add(stake); + }); - // Insert current era staking informations - >::insert(¤t_era, total_staked); - let default_pref = ValidatorPrefs::default(); - for stash in &elected_stashes { - let pref = all_validators_and_prefs.get(stash) - .unwrap_or(&default_pref); // Must never happen, but better to be safe. - >::insert(¤t_era, stash, pref); - } + let exposure = Exposure { + own, + others, + total, + }; - // In order to keep the property required by `n_session_ending` - // that we must return the new validator set even if it's the same as the old, - // as long as any underlying economic conditions have changed, we don't attempt - // to do any optimization where we compare against the prior set. - Some(elected_stashes) - } else { - // There were not enough candidates for even our minimal level of functionality. - // This is bad. - // We should probably disable all functionality except for block production - // and let the chain keep producing blocks until we can decide on a sufficiently - // substantial set. - // TODO: #2494 - None - } + (validator, exposure) + }).collect::)>>() } /// Remove all associated data of a stash account from the staking system. @@ -1949,6 +2825,33 @@ impl Module { Ok(()) } + /// Clear all era information for given era. + fn clear_era_information(era_index: EraIndex) { + >::remove_prefix(era_index); + >::remove_prefix(era_index); + >::remove_prefix(era_index); + >::remove(era_index); + >::remove(era_index); + >::remove(era_index); + ErasStartSessionIndex::remove(era_index); + } + + /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. + fn apply_unapplied_slashes(active_era: EraIndex) { + let slash_defer_duration = T::SlashDeferDuration::get(); + ::EarliestUnappliedSlash::mutate(|earliest| if let Some(ref mut earliest) = earliest { + let keep_from = active_era.saturating_sub(slash_defer_duration); + for era in (*earliest)..keep_from { + let era_slashes = ::UnappliedSlashes::take(&era); + for slash in era_slashes { + slashing::apply_slash::(slash); + } + } + + *earliest = (*earliest).max(keep_from) + }) + } + /// Add reward points to validators using their stash account ID. /// /// Validators are keyed by stash account ID and must be in the current elected set. @@ -1981,6 +2884,16 @@ impl Module { _ => ForceEra::put(Forcing::ForceNew), } } + + #[cfg(feature = "runtime-benchmarks")] + pub fn add_era_stakers(current_era: EraIndex, controller: T::AccountId, exposure: Exposure>) { + >::insert(¤t_era, &controller, &exposure); + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn put_election_status(status: ElectionStatus::) { + >::put(status); + } } /// In this implementation `new_session(session)` must be called before `end_session(session-1)` @@ -2000,9 +2913,7 @@ impl pallet_session::SessionManager for Module { } } -/// This implementation has the same constrains as the implementation of -/// `pallet_session::SessionManager`. -impl SessionManager>> for Module { +impl historical::SessionManager>> for Module { fn new_session(new_index: SessionIndex) -> Option>)>> { @@ -2088,15 +2999,20 @@ impl OnOffenceHandler>], slash_fraction: &[Perbill], slash_session: SessionIndex, - ) { + ) -> Result<(), ()> { + if !Self::can_report() { + return Err(()) + } + let reward_proportion = SlashRewardFraction::get(); let active_era = { let active_era = Self::active_era(); if active_era.is_none() { - return + // this offence need not be re-submitted. + return Ok(()) } - active_era.unwrap().index + active_era.expect("value checked not to be `None`; qed").index }; let active_era_start_session_index = Self::eras_start_session_index(active_era) .unwrap_or_else(|| { @@ -2115,7 +3031,7 @@ impl OnOffenceHandler return, // before bonding period. defensive - should be filtered out. + None => return Ok(()), // before bonding period. defensive - should be filtered out. Some(&(ref slash_era, _)) => *slash_era, } }; @@ -2129,8 +3045,7 @@ impl OnOffenceHandler OnOffenceHandler bool { + Self::era_election_status().is_closed() } } @@ -2191,6 +3112,157 @@ impl ReportOffence } } +/// Disallows any transactions that change the election result to be submitted after the election +/// window is open. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct LockStakingStatus(sp_std::marker::PhantomData); + +impl sp_std::fmt::Debug for LockStakingStatus { + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "LockStakingStatus") + } +} + +impl LockStakingStatus { + /// Create new `LockStakingStatus`. + pub fn new() -> Self { + Self(sp_std::marker::PhantomData) + } +} + +impl Default for LockStakingStatus { + fn default() -> Self { + Self::new() + } +} + +impl SignedExtension for LockStakingStatus { + const IDENTIFIER: &'static str = "LockStakingStatus"; + type AccountId = T::AccountId; + type Call = ::Call; + type AdditionalSigned = (); + type Pre = (); + + fn additional_signed(&self) -> Result<(), TransactionValidityError> { Ok(()) } + + fn validate( + &self, + _who: &Self::AccountId, + call: &Self::Call, + _info: &DispatchInfoOf, + _len: usize, + ) -> TransactionValidity { + if let Some(inner_call) = call.is_sub_type() { + if let ElectionStatus::Open(_) = >::era_election_status() { + match inner_call { + Call::::set_payee(..) | + Call::::set_controller(..) | + Call::::set_validator_count(..) | + Call::::force_no_eras(..) | + Call::::force_new_era(..) | + Call::::set_invulnerables(..) | + Call::::force_unstake(..) | + Call::::force_new_era_always(..) | + Call::::cancel_deferred_slash(..) | + Call::::set_history_depth(..) | + Call::::reap_stash(..) | + Call::::submit_election_solution(..) | + Call::::submit_election_solution_unsigned(..) => { + // These calls are allowed. Nothing. + } + _ => { + return Err(InvalidTransaction::Stale.into()); + } + } + } + } + + Ok(Default::default()) + } +} + +impl From> for InvalidTransaction { + fn from(e: Error) -> Self { + match e { + >::PhragmenEarlySubmission => InvalidTransaction::Future, + _ => InvalidTransaction::Custom(e.as_u8()), + } + } +} + +#[allow(deprecated)] +impl frame_support::unsigned::ValidateUnsigned for Module { + type Call = Call; + fn validate_unsigned(source: TransactionSource, call: &Self::Call) -> TransactionValidity { + if let Call::submit_election_solution_unsigned( + _, + _, + score, + era, + ) = call { + use offchain_election::DEFAULT_LONGEVITY; + + // discard solution not coming from the local OCW. + match source { + TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ } + _ => { + debug::native::debug!( + target: "staking", + "rejecting unsigned transaction because it is not local/in-block." + ); + return InvalidTransaction::Call.into(); + } + } + + if let Err(e) = Self::pre_dispatch_checks(*score, *era) { + debug::native::debug!( + target: "staking", + "validate unsigned failed due to {:?}.", + e, + ); + let invalid: InvalidTransaction = e.into(); + return invalid.into(); + } + + debug::native::debug!( + target: "staking", + "Validated an unsigned transaction from the local node for era {}.", + era, + ); + + ValidTransaction::with_tag_prefix("StakingOffchain") + // The higher the score[0], the better a solution is. + .priority(T::UnsignedPriority::get().saturating_add(score[0].saturated_into())) + // Defensive only. A single solution can exist in the pool per era. Each validator + // will run OCW at most once per era, hence there should never exist more than one + // transaction anyhow. + .and_provides(era) + // Note: this can be more accurate in the future. We do something like + // `era_end_block - current_block` but that is not needed now as we eagerly run + // offchain workers now and the above should be same as `T::ElectionLookahead` + // without the need to query more storage in the validation phase. If we randomize + // offchain worker, then we might re-consider this. + .longevity(TryInto::::try_into( + T::ElectionLookahead::get()).unwrap_or(DEFAULT_LONGEVITY) + ) + // We don't propagate this. This can never the validated at a remote node. + .propagate(false) + .build() + } else { + InvalidTransaction::Call.into() + } + } + + fn pre_dispatch(_: &Self::Call) -> Result<(), TransactionValidityError> { + // IMPORTANT NOTE: By default, a sane `pre-dispatch` should always do the same checks as + // `validate_unsigned` and overriding this should be done with care. this module has only + // one unsigned entry point, in which we call into `>::pre_dispatch_checks()` + // which is all the important checks that we do in `validate_unsigned`. Hence, we can safely + // override this to save some time. + Ok(()) + } +} + /// Check that list is sorted and has no duplicates. fn is_sorted_and_unique(list: &[u32]) -> bool { list.windows(2).all(|w| w[0] < w[1]) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 8754751629731e9b228b589fa91da5a474d73431..6332486b65099fa2eb57b8235ba1cf971df06f48 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,56 +17,72 @@ //! Test utilities use std::{collections::{HashSet, HashMap}, cell::RefCell}; -use sp_runtime::{Perbill, KeyTypeId}; +use sp_runtime::Perbill; use sp_runtime::curve::PiecewiseLinear; -use sp_runtime::traits::{IdentityLookup, Convert, OpaqueKeys, OnInitialize, OnFinalize, SaturatedConversion}; -use sp_runtime::testing::{Header, UintAuthorityId}; +use sp_runtime::traits::{IdentityLookup, Convert, SaturatedConversion, Zero}; +use sp_runtime::testing::{Header, UintAuthorityId, TestXt}; use sp_staking::{SessionIndex, offence::{OffenceDetails, OnOffenceHandler}}; -use sp_core::{H256, crypto::key_types}; -use sp_io; +use sp_core::H256; use frame_support::{ - assert_ok, impl_outer_origin, parameter_types, StorageValue, StorageMap, - StorageDoubleMap, IterableStorageMap, traits::{Currency, Get, FindAuthor}, weights::Weight, + assert_ok, impl_outer_origin, parameter_types, impl_outer_dispatch, impl_outer_event, + StorageValue, StorageMap, StorageDoubleMap, IterableStorageMap, + traits::{Currency, Get, FindAuthor, OnFinalize, OnInitialize}, + weights::Weight, }; -use crate::{ - EraIndex, GenesisConfig, Module, Trait, StakerStatus, ValidatorPrefs, RewardDestination, - Nominators, inflation, SessionInterface, Exposure, ErasStakers, ErasRewardPoints +use frame_system::offchain::TransactionSubmitter; +use sp_io; +use sp_phragmen::{ + build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, PhragmenScore, }; +use crate::*; + +const INIT_TIMESTAMP: u64 = 30_000; /// The AccountId alias in this test module. -pub type AccountId = u64; -pub type BlockNumber = u64; -pub type Balance = u64; +pub(crate) type AccountId = u64; +pub(crate) type AccountIndex = u64; +pub(crate) type BlockNumber = u64; +pub(crate) type Balance = u64; /// Simple structure that exposes how u64 currency can be represented as... u64. pub struct CurrencyToVoteHandler; impl Convert for CurrencyToVoteHandler { - fn convert(x: u64) -> u64 { x } + fn convert(x: u64) -> u64 { + x + } } impl Convert for CurrencyToVoteHandler { - fn convert(x: u128) -> u64 { x.saturated_into() } + fn convert(x: u128) -> u64 { + x.saturated_into() + } } thread_local! { static SESSION: RefCell<(Vec, HashSet)> = RefCell::new(Default::default()); + static SESSION_PER_ERA: RefCell = RefCell::new(3); static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); static SLASH_DEFER_DURATION: RefCell = RefCell::new(0); + static ELECTION_LOOKAHEAD: RefCell = RefCell::new(0); + static PERIOD: RefCell = RefCell::new(1); } -pub struct TestSessionHandler; -impl pallet_session::SessionHandler for TestSessionHandler { - const KEY_TYPE_IDS: &'static [KeyTypeId] = &[key_types::DUMMY]; +/// Another session handler struct to test on_disabled. +pub struct OtherSessionHandler; +impl pallet_session::OneSessionHandler for OtherSessionHandler { + type Key = UintAuthorityId; - fn on_genesis_session(_validators: &[(AccountId, Ks)]) {} + fn on_genesis_session<'a, I: 'a>(_: I) + where I: Iterator, AccountId: 'a {} - fn on_new_session( - _changed: bool, - validators: &[(AccountId, Ks)], - _queued_validators: &[(AccountId, Ks)], - ) { - SESSION.with(|x| - *x.borrow_mut() = (validators.iter().map(|x| x.0.clone()).collect(), HashSet::new()) - ); + fn on_new_session<'a, I: 'a>(_: bool, validators: I, _: I,) + where I: Iterator, AccountId: 'a + { + SESSION.with(|x| { + *x.borrow_mut() = ( + validators.map(|x| x.0.clone()).collect(), + HashSet::new(), + ) + }); } fn on_disabled(validator_index: usize) { @@ -78,6 +94,10 @@ impl pallet_session::SessionHandler for TestSessionHandler { } } +impl sp_runtime::BoundToRuntimeAppPublic for OtherSessionHandler { + type Public = UintAuthorityId; +} + pub fn is_disabled(controller: AccountId) -> bool { let stash = Staking::ledger(&controller).unwrap().stash; SESSION.with(|d| d.borrow().1.contains(&stash)) @@ -90,6 +110,32 @@ impl Get for ExistentialDeposit { } } +pub struct SessionsPerEra; +impl Get for SessionsPerEra { + fn get() -> SessionIndex { + SESSION_PER_ERA.with(|v| *v.borrow()) + } +} +impl Get for SessionsPerEra { + fn get() -> BlockNumber { + SESSION_PER_ERA.with(|v| *v.borrow() as BlockNumber) + } +} + +pub struct ElectionLookahead; +impl Get for ElectionLookahead { + fn get() -> BlockNumber { + ELECTION_LOOKAHEAD.with(|v| *v.borrow()) + } +} + +pub struct Period; +impl Get for Period { + fn get() -> BlockNumber { + PERIOD.with(|v| *v.borrow()) + } +} + pub struct SlashDeferDuration; impl Get for SlashDeferDuration { fn get() -> EraIndex { @@ -97,23 +143,47 @@ impl Get for SlashDeferDuration { } } -impl_outer_origin!{ +impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + staking::Staking, + } +} + +mod staking { + // Re-export needed for `impl_outer_event!`. + pub use super::super::*; +} +use frame_system as system; +use pallet_balances as balances; +use pallet_session as session; + +impl_outer_event! { + pub enum MetaEvent for Test { + system, + balances, + session, + staking, + } +} + /// Author of block is always 11 pub struct Author11; impl FindAuthor for Author11 { fn find_author<'a, I>(_digests: I) -> Option - where I: 'a + IntoIterator + where I: 'a + IntoIterator, { Some(11) } } // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Eq, PartialEq, Debug)] pub struct Test; + parameter_types! { pub const BlockHashCount: u64 = 250; pub const MaximumBlockWeight: Weight = 1024; @@ -122,15 +192,15 @@ parameter_types! { } impl frame_system::Trait for Test { type Origin = Origin; - type Index = u64; + type Index = AccountIndex; type BlockNumber = BlockNumber; - type Call = (); + type Call = Call; type Hash = H256; type Hashing = ::sp_runtime::traits::BlakeTwo256; type AccountId = AccountId; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = MetaEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; @@ -143,26 +213,31 @@ impl frame_system::Trait for Test { } impl pallet_balances::Trait for Test { type Balance = Balance; + type Event = MetaEvent; type DustRemoval = (); - type Event = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; } parameter_types! { - pub const Period: BlockNumber = 1; pub const Offset: BlockNumber = 0; pub const UncleGenerations: u64 = 0; pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(25); } +sp_runtime::impl_opaque_keys! { + pub struct SessionKeys { + pub other: OtherSessionHandler, + } +} impl pallet_session::Trait for Test { - type Event = (); + type SessionManager = pallet_session::historical::NoteHistoricalRoot; + type Keys = SessionKeys; + type ShouldEndSession = pallet_session::PeriodicSessions; + type SessionHandler = (OtherSessionHandler,); + type Event = MetaEvent; type ValidatorId = AccountId; type ValidatorIdOf = crate::StashOf; - type ShouldEndSession = pallet_session::PeriodicSessions; - type SessionManager = pallet_session::historical::NoteHistoricalRoot; - type SessionHandler = TestSessionHandler; - type Keys = UintAuthorityId; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; + type NextSessionRotation = pallet_session::PeriodicSessions; } impl pallet_session::historical::Trait for Test { @@ -194,17 +269,18 @@ pallet_staking_reward_curve::build! { ); } parameter_types! { - pub const SessionsPerEra: SessionIndex = 3; pub const BondingDuration: EraIndex = 3; pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const UnsignedPriority: u64 = 1 << 20; } + impl Trait for Test { - type Currency = pallet_balances::Module; - type Time = pallet_timestamp::Module; + type Currency = Balances; + type UnixTime = Timestamp; type CurrencyToVote = CurrencyToVoteHandler; type RewardRemainder = (); - type Event = (); + type Event = MetaEvent; type Slash = (); type Reward = (); type SessionsPerEra = SessionsPerEra; @@ -213,11 +289,22 @@ impl Trait for Test { type BondingDuration = BondingDuration; type SessionInterface = Self; type RewardCurve = RewardCurve; + type NextNewSession = Session; + type ElectionLookahead = ElectionLookahead; + type Call = Call; + type SubmitTransaction = SubmitTransaction; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = UnsignedPriority; } +pub type Extrinsic = TestXt; +type SubmitTransaction = TransactionSubmitter<(), Test, Extrinsic>; + pub struct ExtBuilder { - existential_deposit: u64, + session_length: BlockNumber, + election_lookahead: BlockNumber, + session_per_era: SessionIndex, + existential_deposit: Balance, validator_pool: bool, nominate: bool, validator_count: u32, @@ -225,13 +312,16 @@ pub struct ExtBuilder { slash_defer_duration: EraIndex, fair: bool, num_validators: Option, - invulnerables: Vec, - stakers: bool, + invulnerables: Vec, + has_stakers: bool, } impl Default for ExtBuilder { fn default() -> Self { Self { + session_length: 1, + election_lookahead: 0, + session_per_era: 3, existential_deposit: 1, validator_pool: false, nominate: true, @@ -241,7 +331,7 @@ impl Default for ExtBuilder { fair: true, num_validators: None, invulnerables: vec![], - stakers: true, + has_stakers: true, } } } @@ -283,18 +373,40 @@ impl ExtBuilder { self.invulnerables = invulnerables; self } - pub fn set_associated_consts(&self) { - EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - SLASH_DEFER_DURATION.with(|v| *v.borrow_mut() = self.slash_defer_duration); + pub fn session_per_era(mut self, length: SessionIndex) -> Self { + self.session_per_era = length; + self } - - pub fn stakers(mut self, has_stakers: bool) -> Self { - self.stakers = has_stakers; + pub fn election_lookahead(mut self, look: BlockNumber) -> Self { + self.election_lookahead = look; + self + } + pub fn session_length(mut self, length: BlockNumber) -> Self { + self.session_length = length; + self + } + pub fn has_stakers(mut self, has: bool) -> Self { + self.has_stakers = has; self } + pub fn offchain_phragmen_ext(self) -> Self { + self.session_per_era(4) + .session_length(5) + .election_lookahead(3) + } + pub fn set_associated_constants(&self) { + EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); + SLASH_DEFER_DURATION.with(|v| *v.borrow_mut() = self.slash_defer_duration); + SESSION_PER_ERA.with(|v| *v.borrow_mut() = self.session_per_era); + ELECTION_LOOKAHEAD.with(|v| *v.borrow_mut() = self.election_lookahead); + PERIOD.with(|v| *v.borrow_mut() = self.session_length); + } pub fn build(self) -> sp_io::TestExternalities { - self.set_associated_consts(); - let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let _ = env_logger::try_init(); + self.set_associated_constants(); + let mut storage = frame_system::GenesisConfig::default() + .build_storage::() + .unwrap(); let balance_factor = if self.existential_deposit > 1 { 256 } else { @@ -306,7 +418,7 @@ impl ExtBuilder { .map(|x| ((x + 1) * 10 + 1) as u64) .collect::>(); - let _ = pallet_balances::GenesisConfig::{ + let _ = pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * balance_factor), (2, 20 * balance_factor), @@ -328,7 +440,7 @@ impl ExtBuilder { }.assimilate_storage(&mut storage); let mut stakers = vec![]; - if self.stakers { + if self.has_stakers { let stake_21 = if self.fair { 1000 } else { 2000 }; let stake_31 = if self.validator_pool { balance_factor * 1000 } else { 1 }; let status_41 = if self.validator_pool { @@ -354,19 +466,31 @@ impl ExtBuilder { invulnerables: self.invulnerables, slash_reward_fraction: Perbill::from_percent(10), ..Default::default() - }.assimilate_storage(&mut storage); + } + .assimilate_storage(&mut storage); let _ = pallet_session::GenesisConfig:: { - keys: validators.iter().map(|x| (*x, *x, UintAuthorityId(*x))).collect(), + keys: validators.iter().map(|x| ( + *x, + *x, + SessionKeys { other: UintAuthorityId(*x) } + )).collect(), }.assimilate_storage(&mut storage); let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { let validators = Session::validators(); - SESSION.with(|x| - *x.borrow_mut() = (validators.clone(), HashSet::new()) - ); + SESSION.with(|x| *x.borrow_mut() = (validators.clone(), HashSet::new())); }); + + // We consider all test to start after timestamp is initialized + // This must be ensured by having `timestamp::on_initialize` called before + // `staking::on_initialize` + ext.execute_with(|| { + System::set_block_number(1); + Timestamp::set_timestamp(INIT_TIMESTAMP); + }); + ext } } @@ -377,6 +501,10 @@ pub type Session = pallet_session::Module; pub type Timestamp = pallet_timestamp::Module; pub type Staking = Module; +pub fn active_era() -> EraIndex { + Staking::active_era().unwrap().index +} + pub fn check_exposure_all(era: EraIndex) { ErasStakers::::iter_prefix(era).for_each(check_exposure) } @@ -389,8 +517,9 @@ pub fn check_nominator_all(era: EraIndex) { /// Check for each selected validator: expo.total = Sum(expo.other) + expo.own pub fn check_exposure(expo: Exposure) { assert_eq!( - expo.total as u128, expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::(), - "wrong total exposure {:?}", expo, + expo.total as u128, + expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::(), + "wrong total exposure", ); } @@ -399,17 +528,18 @@ pub fn check_exposure(expo: Exposure) { pub fn check_nominator_exposure(era: EraIndex, stash: AccountId) { assert_is_stash(stash); let mut sum = 0; - ErasStakers::::iter_prefix(era) - .for_each(|exposure| { - exposure.others.iter() - .filter(|i| i.who == stash) - .for_each(|i| sum += i.value) - }); + Session::validators() + .iter() + .map(|v| Staking::eras_stakers(era, v)) + .for_each(|e| e.others.iter().filter(|i| i.who == stash).for_each(|i| sum += i.value)); let nominator_stake = Staking::slashable_balance_of(&stash); // a nominator cannot over-spend. assert!( nominator_stake >= sum, - "failed: Nominator({}) stake({}) >= sum divided({})", stash, nominator_stake, sum, + "failed: Nominator({}) stake({}) >= sum divided({})", + stash, + nominator_stake, + sum, ); } @@ -425,20 +555,43 @@ pub fn assert_ledger_consistent(stash: AccountId) { assert_eq!(real_total, ledger.total); } -pub fn bond_validator(acc: u64, val: u64) { - // a = controller - // a + 1 = stash - let _ = Balances::make_free_balance_be(&(acc + 1), val); - assert_ok!(Staking::bond(Origin::signed(acc + 1), acc, val, RewardDestination::Controller)); - assert_ok!(Staking::validate(Origin::signed(acc), ValidatorPrefs::default())); -} - -pub fn bond_nominator(acc: u64, val: u64, target: Vec) { - // a = controller - // a + 1 = stash - let _ = Balances::make_free_balance_be(&(acc + 1), val); - assert_ok!(Staking::bond(Origin::signed(acc + 1), acc, val, RewardDestination::Controller)); - assert_ok!(Staking::nominate(Origin::signed(acc), target)); +pub fn bond_validator(stash: u64, ctrl: u64, val: u64) { + let _ = Balances::make_free_balance_be(&stash, val); + let _ = Balances::make_free_balance_be(&ctrl, val); + assert_ok!(Staking::bond( + Origin::signed(stash), + ctrl, + val, + RewardDestination::Controller, + )); + assert_ok!(Staking::validate( + Origin::signed(ctrl), + ValidatorPrefs::default() + )); +} + +pub fn bond_nominator(stash: u64, ctrl: u64, val: u64, target: Vec) { + let _ = Balances::make_free_balance_be(&stash, val); + let _ = Balances::make_free_balance_be(&ctrl, val); + assert_ok!(Staking::bond( + Origin::signed(stash), + ctrl, + val, + RewardDestination::Controller, + )); + assert_ok!(Staking::nominate(Origin::signed(ctrl), target)); +} + +pub fn run_to_block(n: BlockNumber) { + Staking::on_finalize(System::block_number()); + for b in System::block_number() + 1..=n { + System::set_block_number(b); + Session::on_initialize(b); + Staking::on_initialize(b); + if b != n { + Staking::on_finalize(System::block_number()); + } + } } pub fn advance_session() { @@ -447,19 +600,21 @@ pub fn advance_session() { } pub fn start_session(session_index: SessionIndex) { + assert_eq!(>::get(), 1, "start_session can only be used with session length 1."); for i in Session::current_index()..session_index { Staking::on_finalize(System::block_number()); System::set_block_number((i + 1).into()); - Timestamp::set_timestamp(System::block_number() * 1000); + Timestamp::set_timestamp(System::block_number() * 1000 + INIT_TIMESTAMP); Session::on_initialize(System::block_number()); + Staking::on_initialize(System::block_number()); } assert_eq!(Session::current_index(), session_index); } pub fn start_era(era_index: EraIndex) { - start_session((era_index * 3).into()); - assert_eq!(Staking::active_era().unwrap().index, era_index); + start_session((era_index * >::get()).into()); + assert_eq!(Staking::current_era().unwrap(), era_index); } pub fn current_total_payout_for_duration(duration: u64) -> u64 { @@ -472,33 +627,45 @@ pub fn current_total_payout_for_duration(duration: u64) -> u64 { } pub fn reward_all_elected() { - let rewards = ::SessionInterface::validators().into_iter() + let rewards = ::SessionInterface::validators() + .into_iter() .map(|v| (v, 1)); >::reward_by_ids(rewards) } pub fn validator_controllers() -> Vec { - Session::validators().into_iter().map(|s| Staking::bonded(&s).expect("no controller for validator")).collect() + Session::validators() + .into_iter() + .map(|s| Staking::bonded(&s).expect("no controller for validator")) + .collect() } pub fn on_offence_in_era( - offenders: &[OffenceDetails>], + offenders: &[OffenceDetails< + AccountId, + pallet_session::historical::IdentificationTuple, + >], slash_fraction: &[Perbill], era: EraIndex, ) { let bonded_eras = crate::BondedEras::get(); for &(bonded_era, start_session) in bonded_eras.iter() { if bonded_era == era { - Staking::on_offence(offenders, slash_fraction, start_session); - return + let _ = Staking::on_offence(offenders, slash_fraction, start_session).unwrap(); + return; } else if bonded_era > era { - break + break; } } if Staking::active_era().unwrap().index == era { - Staking::on_offence(offenders, slash_fraction, Staking::eras_start_session_index(era).unwrap()); + let _ = + Staking::on_offence( + offenders, + slash_fraction, + Staking::eras_start_session_index(era).unwrap() + ).unwrap(); } else { panic!("cannot slash in era {}", era); } @@ -512,8 +679,193 @@ pub fn on_offence_now( on_offence_in_era(offenders, slash_fraction, now) } +// winners will be chosen by simply their unweighted total backing stake. Nominator stake is +// distributed evenly. +pub fn horrible_phragmen_with_post_processing( + do_reduce: bool, +) -> (CompactAssignments, Vec, PhragmenScore) { + let mut backing_stake_of: BTreeMap = BTreeMap::new(); + + // self stake + >::iter().for_each(|(who, _p)| { + *backing_stake_of.entry(who).or_insert(Zero::zero()) += Staking::slashable_balance_of(&who) + }); + + // add nominator stuff + >::iter().for_each(|(who, nomination)| { + nomination.targets.iter().for_each(|v| { + *backing_stake_of.entry(*v).or_insert(Zero::zero()) += + Staking::slashable_balance_of(&who) + }) + }); + + // elect winners + let mut sorted: Vec = backing_stake_of.keys().cloned().collect(); + sorted.sort_by_key(|x| backing_stake_of.get(x).unwrap()); + let winners: Vec = sorted + .iter() + .cloned() + .take(Staking::validator_count() as usize) + .collect(); + + // create assignments + let mut staked_assignment: Vec> = Vec::new(); + >::iter().for_each(|(who, nomination)| { + let mut dist: Vec<(AccountId, ExtendedBalance)> = Vec::new(); + nomination.targets.iter().for_each(|v| { + if winners.iter().find(|w| *w == v).is_some() { + dist.push((*v, ExtendedBalance::zero())); + } + }); + + if dist.len() == 0 { + return; + } + + // assign real stakes. just split the stake. + let stake = Staking::slashable_balance_of(&who) as ExtendedBalance; + let mut sum: ExtendedBalance = Zero::zero(); + let dist_len = dist.len(); + { + dist.iter_mut().for_each(|(_, w)| { + let partial = stake / (dist_len as ExtendedBalance); + *w = partial; + sum += partial; + }); + } + + // assign the leftover to last. + { + let leftover = stake - sum; + let last = dist.last_mut().unwrap(); + last.1 += leftover; + } + + staked_assignment.push(StakedAssignment { + who, + distribution: dist, + }); + }); + + // Ensure that this result is worse than seq-phragmen. Otherwise, it should not have been used + // for testing. + let score = { + let (_, _, better_score) = prepare_submission_with(true, |_| {}); + + let support = build_support_map::(&winners, &staked_assignment).0; + let score = evaluate_support(&support); + + assert!(sp_phragmen::is_score_better(score, better_score)); + + score + }; + + if do_reduce { + reduce(&mut staked_assignment); + } + + let snapshot_validators = Staking::snapshot_validators().unwrap(); + let snapshot_nominators = Staking::snapshot_nominators().unwrap(); + let nominator_index = |a: &AccountId| -> Option { + snapshot_nominators.iter().position(|x| x == a).map(|i| i as NominatorIndex) + }; + let validator_index = |a: &AccountId| -> Option { + snapshot_validators.iter().position(|x| x == a).map(|i| i as ValidatorIndex) + }; + + // convert back to ratio assignment. This takes less space. + let assignments_reduced = + sp_phragmen::assignment_staked_to_ratio::(staked_assignment); + + let compact = + CompactAssignments::from_assignment(assignments_reduced, nominator_index, validator_index) + .unwrap(); + + // winner ids to index + let winners = winners.into_iter().map(|w| validator_index(&w).unwrap()).collect::>(); + + (compact, winners, score) +} + +// Note: this should always logically reproduce [`offchain_election::prepare_submission`], yet we +// cannot do it since we want to have `tweak` injected into the process. +pub fn prepare_submission_with( + do_reduce: bool, + tweak: impl FnOnce(&mut Vec>), +) -> (CompactAssignments, Vec, PhragmenScore) { + // run phragmen on the default stuff. + let sp_phragmen::PhragmenResult { + winners, + assignments, + } = Staking::do_phragmen::().unwrap(); + let winners = winners.into_iter().map(|(w, _)| w).collect::>(); + + let stake_of = |who: &AccountId| -> ExtendedBalance { + >::convert( + Staking::slashable_balance_of(&who) + ) as ExtendedBalance + }; + let mut staked = sp_phragmen::assignment_ratio_to_staked(assignments, stake_of); + + // apply custom tweaks. awesome for testing. + tweak(&mut staked); + + if do_reduce { + reduce(&mut staked); + } + + // convert back to ratio assignment. This takes less space. + let snapshot_validators = Staking::snapshot_validators().expect("snapshot not created."); + let snapshot_nominators = Staking::snapshot_nominators().expect("snapshot not created."); + let nominator_index = |a: &AccountId| -> Option { + snapshot_nominators + .iter() + .position(|x| x == a) + .map_or_else( + || { println!("unable to find nominator index for {:?}", a); None }, + |i| Some(i as NominatorIndex), + ) + }; + let validator_index = |a: &AccountId| -> Option { + snapshot_validators + .iter() + .position(|x| x == a) + .map_or_else( + || { println!("unable to find validator index for {:?}", a); None }, + |i| Some(i as ValidatorIndex), + ) + }; + + let assignments_reduced = sp_phragmen::assignment_staked_to_ratio(staked); + + // re-compute score by converting, yet again, into staked type + let score = { + let staked = sp_phragmen::assignment_ratio_to_staked( + assignments_reduced.clone(), + Staking::slashable_balance_of_extended, + ); + + let (support_map, _) = build_support_map::( + winners.as_slice(), + staked.as_slice(), + ); + evaluate_support::(&support_map) + }; + + let compact = + CompactAssignments::from_assignment(assignments_reduced, nominator_index, validator_index) + .map_err(|e| { println!("error in compact: {:?}", e); e }) + .expect("Failed to create compact"); + + + // winner ids to index + let winners = winners.into_iter().map(|w| validator_index(&w).unwrap()).collect::>(); + + (compact, winners, score) +} + /// Make all validator and nominator request their payment -pub fn make_all_reward_payment(era: EraIndex) { +pub fn make_all_reward_payment_before_migration(era: EraIndex) { let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() .cloned() .collect::>(); @@ -543,3 +895,37 @@ pub fn make_all_reward_payment(era: EraIndex) { assert_ok!(Staking::payout_validator(Origin::signed(validator_controller), era)); } } + +/// Make all validator and nominator request their payment +pub fn make_all_reward_payment(era: EraIndex) { + let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() + .cloned() + .collect::>(); + + // reward validators + for validator_controller in validators_with_reward.iter().filter_map(Staking::bonded) { + let ledger = >::get(&validator_controller).unwrap(); + + assert_ok!(Staking::payout_stakers(Origin::signed(1337), ledger.stash, era)); + } +} + +#[macro_export] +macro_rules! assert_session_era { + ($session:expr, $era:expr) => { + assert_eq!( + Session::current_index(), + $session, + "wrong session {} != {}", + Session::current_index(), + $session, + ); + assert_eq!( + Staking::active_era().unwrap().index, + $era, + "wrong active era {} != {}", + Staking::active_era().unwrap().index, + $era, + ); + }; +} diff --git a/frame/staking/src/offchain_election.rs b/frame/staking/src/offchain_election.rs new file mode 100644 index 0000000000000000000000000000000000000000..0d4cf49f103b9fd634ad178235159244f4f743b2 --- /dev/null +++ b/frame/staking/src/offchain_election.rs @@ -0,0 +1,219 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Helpers for offchain worker election. + +use crate::{ + Call, CompactAssignments, Module, NominatorIndex, OffchainAccuracy, Trait, ValidatorIndex, +}; +use frame_system::offchain::SubmitUnsignedTransaction; +use sp_phragmen::{ + build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, PhragmenResult, + PhragmenScore, +}; +use sp_runtime::offchain::storage::StorageValueRef; +use sp_runtime::PerThing; +use sp_runtime::RuntimeDebug; +use sp_std::{convert::TryInto, prelude::*}; + +/// Error types related to the offchain election machinery. +#[derive(RuntimeDebug)] +pub enum OffchainElectionError { + /// Phragmen election returned None. This means less candidate that minimum number of needed + /// validators were present. The chain is in trouble and not much that we can do about it. + ElectionFailed, + /// Submission to the transaction pool failed. + PoolSubmissionFailed, + /// The snapshot data is not available. + SnapshotUnavailable, + /// Error from phragmen crate. This usually relates to compact operation. + PhragmenError(sp_phragmen::Error), + /// One of the computed winners is invalid. + InvalidWinner, +} + +impl From for OffchainElectionError { + fn from(e: sp_phragmen::Error) -> Self { + Self::PhragmenError(e) + } +} + +/// Storage key used to store the persistent offchain worker status. +pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/staking-election/"; +/// The repeat threshold of the offchain worker. This means we won't run the offchain worker twice +/// within a window of 5 blocks. +pub(crate) const OFFCHAIN_REPEAT: u32 = 5; +/// Default number of blocks for which the unsigned transaction should stay in the pool +pub(crate) const DEFAULT_LONGEVITY: u64 = 25; + +/// Checks if an execution of the offchain worker is permitted at the given block number, or not. +/// +/// This essentially makes sure that we don't run on previous blocks in case of a re-org, and we +/// don't run twice within a window of length [`OFFCHAIN_REPEAT`]. +/// +/// Returns `Ok(())` if offchain worker should happen, `Err(reason)` otherwise. +pub(crate) fn set_check_offchain_execution_status( + now: T::BlockNumber, +) -> Result<(), &'static str> { + let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); + let threshold = T::BlockNumber::from(OFFCHAIN_REPEAT); + + let mutate_stat = + storage.mutate::<_, &'static str, _>(|maybe_head: Option>| { + match maybe_head { + Some(Some(head)) if now < head => Err("fork."), + Some(Some(head)) if now >= head && now <= head + threshold => { + Err("recently executed.") + } + Some(Some(head)) if now > head + threshold => { + // we can run again now. Write the new head. + Ok(now) + } + _ => { + // value doesn't exists. Probably this node just booted up. Write, and run + Ok(now) + } + } + }); + + match mutate_stat { + // all good + Ok(Ok(_)) => Ok(()), + // failed to write. + Ok(Err(_)) => Err("failed to write to offchain db."), + // fork etc. + Err(why) => Err(why), + } +} + +/// The internal logic of the offchain worker of this module. This runs the phragmen election, +/// compacts and reduces the solution, computes the score and submits it back to the chain as an +/// unsigned transaction, without any signature. +pub(crate) fn compute_offchain_election() -> Result<(), OffchainElectionError> { + // compute raw solution. Note that we use `OffchainAccuracy`. + let PhragmenResult { + winners, + assignments, + } = >::do_phragmen::() + .ok_or(OffchainElectionError::ElectionFailed)?; + + // process and prepare it for submission. + let (winners, compact, score) = prepare_submission::(assignments, winners, true)?; + + // defensive-only: active era can never be none except genesis. + let era = >::active_era().map(|e| e.index).unwrap_or_default(); + + // send it. + let call: ::Call = Call::submit_election_solution_unsigned( + winners, + compact, + score, + era, + ).into(); + + T::SubmitTransaction::submit_unsigned(call) + .map_err(|_| OffchainElectionError::PoolSubmissionFailed) +} + +/// Takes a phragmen result and spits out some data that can be submitted to the chain. +/// +/// This does a lot of stuff; read the inline comments. +pub fn prepare_submission( + assignments: Vec>, + winners: Vec<(T::AccountId, ExtendedBalance)>, + do_reduce: bool, +) -> Result<(Vec, CompactAssignments, PhragmenScore), OffchainElectionError> where + ExtendedBalance: From<::Inner>, +{ + // make sure that the snapshot is available. + let snapshot_validators = + >::snapshot_validators().ok_or(OffchainElectionError::SnapshotUnavailable)?; + let snapshot_nominators = + >::snapshot_nominators().ok_or(OffchainElectionError::SnapshotUnavailable)?; + + // all helper closures + let nominator_index = |a: &T::AccountId| -> Option { + snapshot_nominators + .iter() + .position(|x| x == a) + .and_then(|i| >::try_into(i).ok()) + }; + let validator_index = |a: &T::AccountId| -> Option { + snapshot_validators + .iter() + .position(|x| x == a) + .and_then(|i| >::try_into(i).ok()) + }; + + // Clean winners. + let winners = winners + .into_iter() + .map(|(w, _)| w) + .collect::>(); + + // convert into absolute value and to obtain the reduced version. + let mut staked = sp_phragmen::assignment_ratio_to_staked( + assignments, + >::slashable_balance_of_extended, + ); + + if do_reduce { + reduce(&mut staked); + } + + // Convert back to ratio assignment. This takes less space. + let low_accuracy_assignment = sp_phragmen::assignment_staked_to_ratio(staked); + + // convert back to staked to compute the score in the receiver's accuracy. This can be done + // nicer, for now we do it as such since this code is not time-critical. This ensure that the + // score _predicted_ here is the same as the one computed on chain and you will not get a + // `PhragmenBogusScore` error. This is totally NOT needed if we don't do reduce. This whole + // _accuracy glitch_ happens because reduce breaks that assumption of rounding and **scale**. + // The initial phragmen results are computed in `OffchainAccuracy` and the initial `staked` + // assignment set is also all multiples of this value. After reduce, this no longer holds. Hence + // converting to ratio thereafter is not trivially reversible. + let score = { + let staked = sp_phragmen::assignment_ratio_to_staked( + low_accuracy_assignment.clone(), + >::slashable_balance_of_extended, + ); + + let (support_map, _) = build_support_map::(&winners, &staked); + evaluate_support::(&support_map) + }; + + // compact encode the assignment. + let compact = CompactAssignments::from_assignment( + low_accuracy_assignment, + nominator_index, + validator_index, + ).map_err(|e| OffchainElectionError::from(e))?; + + // winners to index. Use a simple for loop for a more expressive early exit in case of error. + let mut winners_indexed: Vec = Vec::with_capacity(winners.len()); + for w in winners { + if let Some(idx) = snapshot_validators.iter().position(|v| *v == w) { + let compact_index: ValidatorIndex = idx + .try_into() + .map_err(|_| OffchainElectionError::InvalidWinner)?; + winners_indexed.push(compact_index); + } else { + return Err(OffchainElectionError::InvalidWinner); + } + } + + Ok((winners_indexed, compact, score)) +} diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 160c08246aedfa365f3f9ac372bb917081d664bb..26f0828989d733bfc720de288f33f0efcafd462d 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -16,11 +16,11 @@ //! A slashing implementation for NPoS systems. //! -//! For the purposes of the economic model, it is easiest to think of each validator -//! of a nominator which nominates only its own identity. +//! For the purposes of the economic model, it is easiest to think of each validator as a nominator +//! which nominates only its own identity. //! -//! The act of nomination signals intent to unify economic identity with the validator - to take part in the -//! rewards of a job well done, and to take part in the punishment of a job done badly. +//! The act of nomination signals intent to unify economic identity with the validator - to take +//! part in the rewards of a job well done, and to take part in the punishment of a job done badly. //! //! There are 3 main difficulties to account for with slashing in NPoS: //! - A nominator can nominate multiple validators and be slashed via any of them. @@ -52,7 +52,7 @@ use super::{ EraIndex, Trait, Module, Store, BalanceOf, Exposure, Perbill, SessionInterface, NegativeImbalanceOf, UnappliedSlash, }; -use sp_runtime::{traits::{Zero, Saturating}, PerThing}; +use sp_runtime::{traits::{Zero, Saturating}, RuntimeDebug}; use frame_support::{ StorageMap, StorageDoubleMap, traits::{Currency, OnUnbalanced, Imbalance}, @@ -65,7 +65,7 @@ use codec::{Encode, Decode}; const REWARD_F1: Perbill = Perbill::from_percent(50); /// The index of a slashing span - unique to each stash. -pub(crate) type SpanIndex = u32; +pub type SpanIndex = u32; // A range of start..end eras for a slashing span. #[derive(Encode, Decode)] @@ -83,7 +83,7 @@ impl SlashingSpan { } /// An encoding of all of a nominator's slashing spans. -#[derive(Encode, Decode)] +#[derive(Encode, Decode, RuntimeDebug)] pub struct SlashingSpans { // the index of the current slashing span of the nominator. different for // every stash, resets when the account hits free balance 0. @@ -143,7 +143,7 @@ impl SlashingSpans { } /// Yields the era index where the most recent non-zero slash occurred. - pub(crate) fn last_nonzero_slash(&self) -> EraIndex { + pub fn last_nonzero_slash(&self) -> EraIndex { self.last_nonzero_slash } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs new file mode 100644 index 0000000000000000000000000000000000000000..29a395b89de4dda8aa2c56755ed941a52ff99411 --- /dev/null +++ b/frame/staking/src/testing_utils.rs @@ -0,0 +1,340 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Testing utils for staking. Needs the `testing-utils` feature to be enabled. +//! +//! Note that these helpers should NOT be used with the actual crate tests, but are rather designed +//! for when the module is being externally tested (i.e. fuzzing, benchmarking, e2e tests). Enabling +//! this feature in the current crate's Cargo.toml will leak the all of this into a normal release +//! build. Just don't do it. + +use crate::*; +use codec::{Decode, Encode}; +use frame_support::assert_ok; +use frame_system::RawOrigin; +use pallet_indices::address::Address; +use rand::Rng; +use sp_core::hashing::blake2_256; +use sp_phragmen::{ + build_support_map, evaluate_support, reduce, Assignment, PhragmenScore, StakedAssignment, +}; + +const CTRL_PREFIX: u32 = 1000; +const NOMINATOR_PREFIX: u32 = 1_000_000; + +/// A dummy suer. +pub const USER: u32 = 999_999_999; + +/// Address type of the `T` +pub type AddressOf = Address<::AccountId, u32>; + +/// Random number in the range `[a, b]`. +pub fn random(a: u32, b: u32) -> u32 { + rand::thread_rng().gen_range(a, b) +} + +/// Set the desired validator count, with related storage items. +pub fn set_validator_count(to_elect: u32) { + ValidatorCount::put(to_elect); + MinimumValidatorCount::put(to_elect / 2); + >::put(ElectionStatus::Open(T::BlockNumber::from(1u32))); +} + +/// Build an account with the given index. +pub fn account(index: u32) -> T::AccountId { + let entropy = (b"benchmark/staking", index).using_encoded(blake2_256); + T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() +} + +/// Build an address given Index +pub fn address(index: u32) -> AddressOf { + pallet_indices::address::Address::Id(account::(index)) +} + +/// Generate signed origin from `who`. +pub fn signed(who: T::AccountId) -> T::Origin { + RawOrigin::Signed(who).into() +} + +/// Generate signed origin from `index`. +pub fn signed_account(index: u32) -> T::Origin { + signed::(account::(index)) +} + +/// Bond a validator. +pub fn bond_validator(stash: T::AccountId, ctrl: u32, val: BalanceOf) +where + T::Lookup: StaticLookup>, +{ + let _ = T::Currency::make_free_balance_be(&stash, val); + assert_ok!(>::bond( + signed::(stash), + address::(ctrl), + val, + RewardDestination::Controller + )); + assert_ok!(>::validate( + signed_account::(ctrl), + ValidatorPrefs::default() + )); +} + +pub fn bond_nominator( + stash: T::AccountId, + ctrl: u32, + val: BalanceOf, + target: Vec>, +) where + T::Lookup: StaticLookup>, +{ + let _ = T::Currency::make_free_balance_be(&stash, val); + assert_ok!(>::bond( + signed::(stash), + address::(ctrl), + val, + RewardDestination::Controller + )); + assert_ok!(>::nominate(signed_account::(ctrl), target)); +} + +/// Bond `nun_validators` validators and `num_nominator` nominators with `edge_per_voter` random +/// votes per nominator. +pub fn setup_chain_stakers(num_validators: u32, num_voters: u32, edge_per_voter: u32) +where + T::Lookup: StaticLookup>, +{ + (0..num_validators).for_each(|i| { + bond_validator::( + account::(i), + i + CTRL_PREFIX, + >::from(random(1, 1000)) * T::Currency::minimum_balance(), + ); + }); + + (0..num_voters).for_each(|i| { + let mut targets: Vec> = Vec::with_capacity(edge_per_voter as usize); + let mut all_targets = (0..num_validators) + .map(|t| address::(t)) + .collect::>(); + assert!(num_validators >= edge_per_voter); + (0..edge_per_voter).for_each(|_| { + let target = all_targets.remove(random(0, all_targets.len() as u32 - 1) as usize); + targets.push(target); + }); + bond_nominator::( + account::(i + NOMINATOR_PREFIX), + i + NOMINATOR_PREFIX + CTRL_PREFIX, + >::from(random(1, 1000)) * T::Currency::minimum_balance(), + targets, + ); + }); + + >::create_stakers_snapshot(); +} + +/// Build a _really bad_ but acceptable solution for election. This should always yield a solution +/// which has a less score than the seq-phragmen. +pub fn get_weak_solution( + do_reduce: bool, +) -> (Vec, CompactAssignments, PhragmenScore) { + let mut backing_stake_of: BTreeMap> = BTreeMap::new(); + + // self stake + >::enumerate().for_each(|(who, _p)| { + *backing_stake_of.entry(who.clone()).or_insert(Zero::zero()) += + >::slashable_balance_of(&who) + }); + + // add nominator stuff + >::enumerate().for_each(|(who, nomination)| { + nomination.targets.into_iter().for_each(|v| { + *backing_stake_of.entry(v).or_insert(Zero::zero()) += + >::slashable_balance_of(&who) + }) + }); + + // elect winners + let mut sorted: Vec = backing_stake_of.keys().cloned().collect(); + sorted.sort_by_key(|x| backing_stake_of.get(x).unwrap()); + let winners: Vec = sorted + .iter() + .cloned() + .take(>::validator_count() as usize) + .collect(); + + let mut staked_assignments: Vec> = Vec::new(); + >::enumerate().for_each(|(who, nomination)| { + let mut dist: Vec<(T::AccountId, ExtendedBalance)> = Vec::new(); + nomination.targets.into_iter().for_each(|v| { + if winners.iter().find(|&w| *w == v).is_some() { + dist.push((v, ExtendedBalance::zero())); + } + }); + + if dist.len() == 0 { + return; + } + + // assign real stakes. just split the stake. + let stake = , u64>>::convert( + >::slashable_balance_of(&who), + ) as ExtendedBalance; + + let mut sum: ExtendedBalance = Zero::zero(); + let dist_len = dist.len() as ExtendedBalance; + + // assign main portion + // only take the first half into account. This should highly imbalance stuff, which is good. + dist.iter_mut() + .take(if dist_len > 1 { + (dist_len as usize) / 2 + } else { + 1 + }) + .for_each(|(_, w)| { + let partial = stake / dist_len; + *w = partial; + sum += partial; + }); + + // assign the leftover to last. + let leftover = stake - sum; + let last = dist.last_mut().unwrap(); + last.1 += leftover; + + staked_assignments.push(StakedAssignment { + who, + distribution: dist, + }); + }); + + // add self support to winners. + winners.iter().for_each(|w| { + staked_assignments.push(StakedAssignment { + who: w.clone(), + distribution: vec![( + w.clone(), + , u64>>::convert( + >::slashable_balance_of(&w), + ) as ExtendedBalance, + )], + }) + }); + + if do_reduce { + reduce(&mut staked_assignments); + } + + // helpers for building the compact + let snapshot_validators = >::snapshot_validators().unwrap(); + let snapshot_nominators = >::snapshot_nominators().unwrap(); + + let nominator_index = |a: &T::AccountId| -> Option { + snapshot_nominators + .iter() + .position(|x| x == a) + .and_then(|i| >::try_into(i).ok()) + }; + let validator_index = |a: &T::AccountId| -> Option { + snapshot_validators + .iter() + .position(|x| x == a) + .and_then(|i| >::try_into(i).ok()) + }; + let stake_of = |who: &T::AccountId| -> ExtendedBalance { + , u64>>::convert( + >::slashable_balance_of(who), + ) as ExtendedBalance + }; + + // convert back to ratio assignment. This takes less space. + let low_accuracy_assignment: Vec> = + staked_assignments + .into_iter() + .map(|sa| sa.into_assignment(true)) + .collect(); + + // re-calculate score based on what the chain will decode. + let score = { + let staked: Vec> = low_accuracy_assignment + .iter() + .map(|a| { + let stake = stake_of(&a.who); + a.clone().into_staked(stake, true) + }) + .collect(); + + let (support_map, _) = + build_support_map::(winners.as_slice(), staked.as_slice()); + evaluate_support::(&support_map) + }; + + // compact encode the assignment. + let compact = CompactAssignments::from_assignment( + low_accuracy_assignment, + nominator_index, + validator_index, + ) + .unwrap(); + + // winners to index. + let winners = winners + .into_iter() + .map(|w| { + snapshot_validators + .iter() + .position(|v| *v == w) + .unwrap() + .try_into() + .unwrap() + }) + .collect::>(); + + (winners, compact, score) +} + +/// Create a solution for seq-phragmen. This uses the same internal function as used by the offchain +/// worker code. +pub fn get_seq_phragmen_solution( + do_reduce: bool, +) -> (Vec, CompactAssignments, PhragmenScore) { + let sp_phragmen::PhragmenResult { + winners, + assignments, + } = >::do_phragmen::().unwrap(); + + offchain_election::prepare_submission::(assignments, winners, do_reduce).unwrap() +} + +/// Remove all validator, nominators, votes and exposures. +pub fn clean(era: EraIndex) + where + ::AccountId: codec::EncodeLike, + u32: codec::EncodeLike, +{ + >::enumerate().for_each(|(k, _)| { + let ctrl = >::bonded(&k).unwrap(); + >::remove(&k); + >::remove(&k); + >::remove(&ctrl); + >::remove(k, era); + }); + >::enumerate().for_each(|(k, _)| >::remove(k)); + >::remove_all(); + >::remove_all(); + >::kill(); + QueuedScore::kill(); +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index d11457671f7c2a4cd991bdd61a717cc61191faf0..321fcb34ef8f0526d303b7e9512983126184ccdc 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -18,12 +18,13 @@ use super::*; use mock::*; -use sp_runtime::{assert_eq_error_rate, traits::{OnInitialize, BadOrigin}}; +use sp_runtime::{ + assert_eq_error_rate, traits::BadOrigin, +}; use sp_staking::offence::OffenceDetails; use frame_support::{ - assert_ok, assert_noop, - traits::{Currency, ReservableCurrency}, - StorageMap, + assert_ok, assert_noop, StorageMap, + traits::{Currency, ReservableCurrency, OnInitialize}, }; use pallet_balances::Error as BalancesError; use substrate_test_utils::assert_eq_uvec; @@ -65,12 +66,12 @@ fn basic_setup_works() { // Account 10 controls the stash from account 11, which is 100 * balance_factor units assert_eq!( Staking::ledger(&10), - Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], last_reward: None }) + Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: vec![] }) ); // Account 20 controls the stash from account 21, which is 200 * balance_factor units assert_eq!( Staking::ledger(&20), - Some(StakingLedger { stash: 21, total: 1000, active: 1000, unlocking: vec![], last_reward: None }) + Some(StakingLedger { stash: 21, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: vec![] }) ); // Account 1 does not control any stash assert_eq!(Staking::ledger(&1), None); @@ -84,7 +85,7 @@ fn basic_setup_works() { assert_eq!( Staking::ledger(100), - Some(StakingLedger { stash: 101, total: 500, active: 500, unlocking: vec![], last_reward: None }) + Some(StakingLedger { stash: 101, total: 500, active: 500, unlocking: vec![], claimed_rewards: vec![] }) ); assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); @@ -139,7 +140,7 @@ fn change_controller_works() { assert_ok!(Staking::set_controller(Origin::signed(11), 5)); - start_era(1); + mock::start_era(1); assert_noop!( Staking::validate(Origin::signed(10), ValidatorPrefs::default()), @@ -221,7 +222,7 @@ fn rewards_should_work() { let total_payout_1 = current_total_payout_for_duration(3 * 1000); assert!(total_payout_1 > 10); // Test is meaningful if reward something - start_era(2); + mock::start_era(2); mock::make_all_reward_payment(1); assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); @@ -253,8 +254,6 @@ fn staking_should_work() { // --- Block 1: start_session(1); - Timestamp::set_timestamp(1); // Initialize time. - // remember + compare this along with the test. assert_eq_uvec!(validator_controllers(), vec![20, 10]); @@ -265,7 +264,6 @@ fn staking_should_work() { start_session(2); // add a new candidate for being a validator. account 3 controlled by 4. assert_ok!(Staking::bond(Origin::signed(3), 4, 1500, RewardDestination::Controller)); - let current_era_at_bond = Staking::current_era(); assert_ok!(Staking::validate(Origin::signed(4), ValidatorPrefs::default())); // No effects will be seen so far. @@ -312,7 +310,7 @@ fn staking_should_work() { total: 1500, active: 1500, unlocking: vec![], - last_reward: current_era_at_bond, + claimed_rewards: vec![0], }) ); // e.g. it cannot spend more than 500 that it has free from the total 2000 @@ -337,7 +335,7 @@ fn less_than_needed_candidates_works() { assert_eq!(Staking::minimum_validator_count(), 1); assert_eq_uvec!(validator_controllers(), vec![30, 20, 10]); - start_era(1); + mock::start_era(1); // Previous set is selected. NO election algorithm is even executed. assert_eq_uvec!(validator_controllers(), vec![30, 20, 10]); @@ -375,7 +373,7 @@ fn no_candidate_emergency_condition() { let _ = Staking::chill(Origin::signed(10)); // trigger era - start_era(1); + mock::start_era(1); // Previous ones are elected. chill is invalidates. TODO: #2494 assert_eq_uvec!(validator_controllers(), vec![10, 20, 30, 40]); @@ -455,7 +453,7 @@ fn nominating_and_rewards_should_work() { >::reward_by_ids(vec![(41, 1)]); >::reward_by_ids(vec![(31, 1)]); - start_era(1); + mock::start_era(1); // 10 and 20 have more votes, they will be chosen by phragmen. assert_eq_uvec!(validator_controllers(), vec![20, 10]); @@ -498,7 +496,7 @@ fn nominating_and_rewards_should_work() { >::reward_by_ids(vec![(21, 2)]); >::reward_by_ids(vec![(11, 1)]); - start_era(2); + mock::start_era(2); // nothing else will happen, era ends and rewards are paid again, // it is expected that nominators will also be paid. See below @@ -566,7 +564,7 @@ fn nominators_also_get_slashed() { >::reward_by_ids(vec![(11, 1)]); // new era, pay rewards, - start_era(1); + mock::start_era(1); // Nominator stash didn't collect any. assert_eq!(Balances::total_balance(&2), initial_balance); @@ -648,51 +646,37 @@ fn double_controlling_should_fail() { fn session_and_eras_work() { ExtBuilder::default().build().execute_with(|| { assert_eq!(Staking::active_era().unwrap().index, 0); + assert_eq!(Session::current_index(), 0); - // Block 1: No change. + // Session 1: No change. start_session(1); assert_eq!(Session::current_index(), 1); assert_eq!(Staking::active_era().unwrap().index, 0); - // Block 2: No change. + // Session 2: No change. start_session(2); assert_eq!(Session::current_index(), 2); assert_eq!(Staking::active_era().unwrap().index, 0); - // Block 3: Era increment. + // Session 3: Era increment. start_session(3); assert_eq!(Session::current_index(), 3); assert_eq!(Staking::active_era().unwrap().index, 1); - // Block 4: No change. + // Session 4: No change. start_session(4); assert_eq!(Session::current_index(), 4); assert_eq!(Staking::active_era().unwrap().index, 1); - // Block 5: No change. + // Session 5: No change. start_session(5); assert_eq!(Session::current_index(), 5); assert_eq!(Staking::active_era().unwrap().index, 1); - // Block 6: Era increment. + // Session 6: Era increment. start_session(6); assert_eq!(Session::current_index(), 6); assert_eq!(Staking::active_era().unwrap().index, 2); - - // Block 7: No change. - start_session(7); - assert_eq!(Session::current_index(), 7); - assert_eq!(Staking::active_era().unwrap().index, 2); - - // Block 8: No change. - start_session(8); - assert_eq!(Session::current_index(), 8); - assert_eq!(Staking::active_era().unwrap().index, 2); - - // Block 9: Era increment. - start_session(9); - assert_eq!(Session::current_index(), 9); - assert_eq!(Staking::active_era().unwrap().index, 3); }); } @@ -834,7 +818,7 @@ fn reward_destination_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); // Compute total payout now for whole duration as other parameter won't change @@ -842,7 +826,7 @@ fn reward_destination_works() { assert!(total_payout_0 > 100); // Test is meaningful if reward something >::reward_by_ids(vec![(11, 1)]); - start_era(1); + mock::start_era(1); mock::make_all_reward_payment(0); // Check that RewardDestination is Staked (default) @@ -855,7 +839,7 @@ fn reward_destination_works() { total: 1000 + total_payout_0, active: 1000 + total_payout_0, unlocking: vec![], - last_reward: Some(0), + claimed_rewards: vec![0], })); //Change RewardDestination to Stash @@ -866,7 +850,7 @@ fn reward_destination_works() { assert!(total_payout_1 > 100); // Test is meaningful if reward something >::reward_by_ids(vec![(11, 1)]); - start_era(2); + mock::start_era(2); mock::make_all_reward_payment(1); // Check that RewardDestination is Stash @@ -881,7 +865,7 @@ fn reward_destination_works() { total: 1000 + total_payout_0, active: 1000 + total_payout_0, unlocking: vec![], - last_reward: Some(1), + claimed_rewards: vec![0,1], })); // Change RewardDestination to Controller @@ -895,7 +879,7 @@ fn reward_destination_works() { assert!(total_payout_2 > 100); // Test is meaningful if reward something >::reward_by_ids(vec![(11, 1)]); - start_era(3); + mock::start_era(3); mock::make_all_reward_payment(2); // Check that RewardDestination is Controller @@ -908,7 +892,7 @@ fn reward_destination_works() { total: 1000 + total_payout_0, active: 1000 + total_payout_0, unlocking: vec![], - last_reward: Some(2), + claimed_rewards: vec![0,1,2], })); // Check that amount in staked account is NOT increased. assert_eq!(Balances::free_balance(11), recorded_stash_balance); @@ -930,7 +914,7 @@ fn validator_payment_prefs_work() { >::insert(&11, RewardDestination::Controller); >::insert(&101, RewardDestination::Controller); - start_era(1); + mock::start_era(1); mock::make_all_reward_payment(0); let balance_era_1_10 = Balances::total_balance(&10); @@ -942,7 +926,7 @@ fn validator_payment_prefs_work() { let exposure_1 = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); >::reward_by_ids(vec![(11, 1)]); - start_era(2); + mock::start_era(2); mock::make_all_reward_payment(1); let taken_cut = commission * total_payout_1; @@ -974,7 +958,7 @@ fn bond_extra_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); // Give account 11 some large free balance greater than total @@ -988,7 +972,7 @@ fn bond_extra_works() { total: 1000 + 100, active: 1000 + 100, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); // Call the bond_extra function with a large number, should handle it @@ -999,7 +983,7 @@ fn bond_extra_works() { total: 1000000, active: 1000000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); }); } @@ -1026,7 +1010,7 @@ fn bond_extra_and_withdraw_unbonded_works() { assert_eq!(Balances::total_balance(&10), 1); // confirm that 10 is a normal validator and gets paid at the end of the era. - start_era(1); + mock::start_era(1); // Initial state of 10 assert_eq!(Staking::ledger(&10), Some(StakingLedger { @@ -1034,7 +1018,7 @@ fn bond_extra_and_withdraw_unbonded_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11), Exposure { total: 1000, own: 1000, others: vec![] }); @@ -1046,14 +1030,13 @@ fn bond_extra_and_withdraw_unbonded_works() { total: 1000 + 100, active: 1000 + 100, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); // Exposure is a snapshot! only updated after the next era update. assert_ne!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11), Exposure { total: 1000 + 100, own: 1000 + 100, others: vec![] }); // trigger next era. - Timestamp::set_timestamp(10); - start_era(2); + mock::start_era(2); assert_eq!(Staking::active_era().unwrap().index, 2); // ledger should be the same. @@ -1062,7 +1045,7 @@ fn bond_extra_and_withdraw_unbonded_works() { total: 1000 + 100, active: 1000 + 100, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], })); // Exposure is now updated. assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11), Exposure { total: 1000 + 100, own: 1000 + 100, others: vec![] }); @@ -1070,29 +1053,29 @@ fn bond_extra_and_withdraw_unbonded_works() { // Unbond almost all of the funds in stash. Staking::unbond(Origin::signed(10), 1000).unwrap(); assert_eq!(Staking::ledger(&10), Some(StakingLedger { - stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], last_reward: None }) + stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], claimed_rewards: vec![] }) ); // Attempting to free the balances now will fail. 2 eras need to pass. Staking::withdraw_unbonded(Origin::signed(10)).unwrap(); assert_eq!(Staking::ledger(&10), Some(StakingLedger { - stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], last_reward: None })); + stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], claimed_rewards: vec![] })); // trigger next era. - start_era(3); + mock::start_era(3); // nothing yet Staking::withdraw_unbonded(Origin::signed(10)).unwrap(); assert_eq!(Staking::ledger(&10), Some(StakingLedger { - stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], last_reward: None })); + stash: 11, total: 1000 + 100, active: 100, unlocking: vec![UnlockChunk{ value: 1000, era: 2 + 3}], claimed_rewards: vec![] })); // trigger next era. - start_era(5); + mock::start_era(5); Staking::withdraw_unbonded(Origin::signed(10)).unwrap(); // Now the value is free and the staking ledger is updated. assert_eq!(Staking::ledger(&10), Some(StakingLedger { - stash: 11, total: 100, active: 100, unlocking: vec![], last_reward: None })); + stash: 11, total: 100, active: 100, unlocking: vec![], claimed_rewards: vec![] })); }) } @@ -1104,14 +1087,14 @@ fn too_many_unbond_calls_should_not_work() { assert_ok!(Staking::unbond(Origin::signed(10), 1)); } - start_era(1); + mock::start_era(1); // locked at era 1 until 4 assert_ok!(Staking::unbond(Origin::signed(10), 1)); // can't do more. assert_noop!(Staking::unbond(Origin::signed(10), 1), Error::::NoMoreChunks); - start_era(3); + mock::start_era(3); assert_noop!(Staking::unbond(Origin::signed(10), 1), Error::::NoMoreChunks); // free up. @@ -1143,7 +1126,7 @@ fn rebond_works() { let _ = Balances::make_free_balance_be(&11, 1000000); // confirm that 10 is a normal validator and gets paid at the end of the era. - start_era(1); + mock::start_era(1); // Initial state of 10 assert_eq!( @@ -1153,11 +1136,11 @@ fn rebond_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], }) ); - start_era(2); + mock::start_era(2); assert_eq!(Staking::active_era().unwrap().index, 2); // Try to rebond some funds. We get an error since no fund is unbonded. @@ -1178,7 +1161,7 @@ fn rebond_works() { value: 900, era: 2 + 3, }], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1191,7 +1174,7 @@ fn rebond_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1204,7 +1187,7 @@ fn rebond_works() { total: 1000, active: 100, unlocking: vec![UnlockChunk { value: 900, era: 5 }], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1217,7 +1200,7 @@ fn rebond_works() { total: 1000, active: 600, unlocking: vec![UnlockChunk { value: 400, era: 5 }], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1230,7 +1213,7 @@ fn rebond_works() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1249,7 +1232,7 @@ fn rebond_works() { UnlockChunk { value: 300, era: 5 }, UnlockChunk { value: 300, era: 5 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1265,7 +1248,7 @@ fn rebond_works() { UnlockChunk { value: 300, era: 5 }, UnlockChunk { value: 100, era: 5 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); }) @@ -1288,7 +1271,7 @@ fn rebond_is_fifo() { let _ = Balances::make_free_balance_be(&11, 1000000); // confirm that 10 is a normal validator and gets paid at the end of the era. - start_era(1); + mock::start_era(1); // Initial state of 10 assert_eq!( @@ -1298,11 +1281,11 @@ fn rebond_is_fifo() { total: 1000, active: 1000, unlocking: vec![], - last_reward: None, + claimed_rewards: vec![], }) ); - start_era(2); + mock::start_era(2); // Unbond some of the funds in stash. Staking::unbond(Origin::signed(10), 400).unwrap(); @@ -1315,11 +1298,11 @@ fn rebond_is_fifo() { unlocking: vec![ UnlockChunk { value: 400, era: 2 + 3 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); - start_era(3); + mock::start_era(3); // Unbond more of the funds in stash. Staking::unbond(Origin::signed(10), 300).unwrap(); @@ -1333,11 +1316,11 @@ fn rebond_is_fifo() { UnlockChunk { value: 400, era: 2 + 3 }, UnlockChunk { value: 300, era: 3 + 3 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); - start_era(4); + mock::start_era(4); // Unbond yet more of the funds in stash. Staking::unbond(Origin::signed(10), 200).unwrap(); @@ -1352,7 +1335,7 @@ fn rebond_is_fifo() { UnlockChunk { value: 300, era: 3 + 3 }, UnlockChunk { value: 200, era: 4 + 3 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); @@ -1368,7 +1351,7 @@ fn rebond_is_fifo() { UnlockChunk { value: 400, era: 2 + 3 }, UnlockChunk { value: 100, era: 3 + 3 }, ], - last_reward: None, + claimed_rewards: vec![], }) ); }) @@ -1394,7 +1377,7 @@ fn reward_to_stake_works() { // Now lets lower account 20 stake assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 21).total, 69); - >::insert(&20, StakingLedger { stash: 21, total: 69, active: 69, unlocking: vec![], last_reward: None }); + >::insert(&20, StakingLedger { stash: 21, total: 69, active: 69, unlocking: vec![], claimed_rewards: vec![] }); // Compute total payout now for whole duration as other parameter won't change let total_payout_0 = current_total_payout_for_duration(3000); @@ -1403,7 +1386,7 @@ fn reward_to_stake_works() { >::reward_by_ids(vec![(21, 1)]); // New era --> rewards are paid --> stakes are changed - start_era(1); + mock::start_era(1); mock::make_all_reward_payment(0); assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11).total, 1000); @@ -1413,7 +1396,7 @@ fn reward_to_stake_works() { assert_eq!(_11_balance, 1000 + total_payout_0 / 2); // Trigger another new era as the info are frozen before the era start. - start_era(2); + mock::start_era(2); // -- new infos assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11).total, 1000 + total_payout_0 / 2); @@ -1557,7 +1540,7 @@ fn switching_roles() { assert_ok!(Staking::bond(Origin::signed(5), 6, 1000, RewardDestination::Controller)); assert_ok!(Staking::validate(Origin::signed(6), ValidatorPrefs::default())); - start_era(1); + mock::start_era(1); // with current nominators 10 and 5 have the most stake assert_eq_uvec!(validator_controllers(), vec![6, 10]); @@ -1571,7 +1554,7 @@ fn switching_roles() { // 2 : 2000 self vote + 250 vote. // Winners: 20 and 2 - start_era(2); + mock::start_era(2); assert_eq_uvec!(validator_controllers(), vec![2, 20]); @@ -1596,7 +1579,7 @@ fn wrong_vote_is_null() { ])); // new block - start_era(1); + mock::start_era(1); assert_eq_uvec!(validator_controllers(), vec![20, 10]); }); @@ -1620,7 +1603,6 @@ fn bond_with_no_staked_value() { ); // bonded with absolute minimum value possible. assert_ok!(Staking::bond(Origin::signed(1), 2, 5, RewardDestination::Controller)); - let current_era_at_bond = Staking::current_era(); assert_eq!(Balances::locks(&1)[0].amount, 5); // unbonding even 1 will cause all to be unbonded. @@ -1632,19 +1614,19 @@ fn bond_with_no_staked_value() { active: 0, total: 5, unlocking: vec![UnlockChunk {value: 5, era: 3}], - last_reward: current_era_at_bond, + claimed_rewards: vec![], }) ); - start_era(1); - start_era(2); + mock::start_era(1); + mock::start_era(2); // not yet removed. assert_ok!(Staking::withdraw_unbonded(Origin::signed(2))); assert!(Staking::ledger(2).is_some()); assert_eq!(Balances::locks(&1)[0].amount, 5); - start_era(3); + mock::start_era(3); // poof. Account 1 is removed from the staking system. assert_ok!(Staking::withdraw_unbonded(Origin::signed(2))); @@ -1677,7 +1659,7 @@ fn bond_with_little_staked_value_bounded() { let total_payout_0 = current_total_payout_for_duration(3000); assert!(total_payout_0 > 100); // Test is meaningful if reward something reward_all_elected(); - start_era(1); + mock::start_era(1); mock::make_all_reward_payment(0); // 2 is elected. @@ -1694,7 +1676,7 @@ fn bond_with_little_staked_value_bounded() { let total_payout_1 = current_total_payout_for_duration(3000); assert!(total_payout_1 > 100); // Test is meaningful if reward something reward_all_elected(); - start_era(2); + mock::start_era(2); mock::make_all_reward_payment(1); assert_eq_uvec!(validator_controllers(), vec![20, 10, 2]); @@ -1722,7 +1704,6 @@ fn new_era_elects_correct_number_of_validators() { assert_eq!(Staking::validator_count(), 1); assert_eq!(validator_controllers().len(), 1); - System::set_block_number(1); Session::on_initialize(System::block_number()); assert_eq!(validator_controllers().len(), 1); @@ -1737,13 +1718,13 @@ fn phragmen_should_not_overflow_validators() { let _ = Staking::chill(Origin::signed(10)); let _ = Staking::chill(Origin::signed(20)); - bond_validator(2, u64::max_value()); - bond_validator(4, u64::max_value()); + bond_validator(3, 2, u64::max_value()); + bond_validator(5, 4, u64::max_value()); - bond_nominator(6, u64::max_value() / 2, vec![3, 5]); - bond_nominator(8, u64::max_value() / 2, vec![3, 5]); + bond_nominator(7, 6, u64::max_value() / 2, vec![3, 5]); + bond_nominator(9, 8, u64::max_value() / 2, vec![3, 5]); - start_era(1); + mock::start_era(1); assert_eq_uvec!(validator_controllers(), vec![4, 2]); @@ -1760,13 +1741,13 @@ fn phragmen_should_not_overflow_nominators() { let _ = Staking::chill(Origin::signed(10)); let _ = Staking::chill(Origin::signed(20)); - bond_validator(2, u64::max_value() / 2); - bond_validator(4, u64::max_value() / 2); + bond_validator(3, 2, u64::max_value() / 2); + bond_validator(5, 4, u64::max_value() / 2); - bond_nominator(6, u64::max_value(), vec![3, 5]); - bond_nominator(8, u64::max_value(), vec![3, 5]); + bond_nominator(7, 6, u64::max_value(), vec![3, 5]); + bond_nominator(9, 8, u64::max_value(), vec![3, 5]); - start_era(1); + mock::start_era(1); assert_eq_uvec!(validator_controllers(), vec![4, 2]); @@ -1779,13 +1760,13 @@ fn phragmen_should_not_overflow_nominators() { #[test] fn phragmen_should_not_overflow_ultimate() { ExtBuilder::default().nominate(false).build().execute_with(|| { - bond_validator(2, u64::max_value()); - bond_validator(4, u64::max_value()); + bond_validator(3, 2, u64::max_value()); + bond_validator(5, 4, u64::max_value()); - bond_nominator(6, u64::max_value(), vec![3, 5]); - bond_nominator(8, u64::max_value(), vec![3, 5]); + bond_nominator(7, 6, u64::max_value(), vec![3, 5]); + bond_nominator(9, 8, u64::max_value(), vec![3, 5]); - start_era(1); + mock::start_era(1); assert_eq_uvec!(validator_controllers(), vec![4, 2]); @@ -1818,7 +1799,7 @@ fn reward_validator_slashing_validator_doesnt_overflow() { ErasStakers::::insert(0, 11, &exposure); ErasStakersClipped::::insert(0, 11, exposure); ErasValidatorReward::::insert(0, stake); - assert_ok!(Staking::payout_validator(Origin::signed(10), 0)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 0)); assert_eq!(Balances::total_balance(&11), stake * 2); // Set staker @@ -1924,21 +1905,23 @@ fn era_is_always_same_length() { // This ensures that the sessions is always of the same length if there is no forcing no // session changes. ExtBuilder::default().build().execute_with(|| { - start_era(1); - assert_eq!(Staking::eras_start_session_index(Staking::active_era().unwrap().index).unwrap(), SessionsPerEra::get()); + let session_per_era = >::get(); + + mock::start_era(1); + assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session_per_era); - start_era(2); - assert_eq!(Staking::eras_start_session_index(Staking::active_era().unwrap().index).unwrap(), SessionsPerEra::get() * 2); + mock::start_era(2); + assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session_per_era * 2u32); let session = Session::current_index(); ForceEra::put(Forcing::ForceNew); advance_session(); advance_session(); assert_eq!(Staking::active_era().unwrap().index, 3); - assert_eq!(Staking::eras_start_session_index(Staking::active_era().unwrap().index).unwrap(), session + 2); + assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session + 2); - start_era(4); - assert_eq!(Staking::eras_start_session_index(Staking::active_era().unwrap().index).unwrap(), session + 2 + SessionsPerEra::get()); + mock::start_era(4); + assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session + 2u32 + session_per_era); }); } @@ -1964,6 +1947,7 @@ fn offence_forces_new_era() { fn offence_ensures_new_era_without_clobbering() { ExtBuilder::default().build().execute_with(|| { assert_ok!(Staking::force_new_era_always(Origin::ROOT)); + assert_eq!(Staking::force_era(), Forcing::ForceAlways); on_offence_now( &[OffenceDetails { @@ -1981,10 +1965,11 @@ fn offence_ensures_new_era_without_clobbering() { } #[test] -fn offence_deselects_validator_when_slash_is_zero() { +fn offence_deselects_validator_even_when_slash_is_zero() { ExtBuilder::default().build().execute_with(|| { assert!(Session::validators().contains(&11)); assert!(>::contains_key(11)); + on_offence_now( &[OffenceDetails { offender: ( @@ -1995,9 +1980,12 @@ fn offence_deselects_validator_when_slash_is_zero() { }], &[Perbill::from_percent(0)], ); + assert_eq!(Staking::force_era(), Forcing::ForceNew); assert!(!>::contains_key(11)); - start_era(1); + + mock::start_era(1); + assert!(!Session::validators().contains(&11)); assert!(!>::contains_key(11)); }); @@ -2034,10 +2022,11 @@ fn slashing_performed_according_exposure() { #[test] fn slash_in_old_span_does_not_deselect() { ExtBuilder::default().build().execute_with(|| { - start_era(1); + mock::start_era(1); assert!(>::contains_key(11)); assert!(Session::validators().contains(&11)); + on_offence_now( &[OffenceDetails { offender: ( @@ -2048,17 +2037,18 @@ fn slash_in_old_span_does_not_deselect() { }], &[Perbill::from_percent(0)], ); + assert_eq!(Staking::force_era(), Forcing::ForceNew); assert!(!>::contains_key(11)); - start_era(2); + mock::start_era(2); Staking::validate(Origin::signed(10), Default::default()).unwrap(); assert_eq!(Staking::force_era(), Forcing::NotForcing); assert!(>::contains_key(11)); assert!(!Session::validators().contains(&11)); - start_era(3); + mock::start_era(3); // this staker is in a new slashing span now, having re-registered after // their prior slash. @@ -2075,7 +2065,7 @@ fn slash_in_old_span_does_not_deselect() { 1, ); - // not for zero-slash. + // not forcing for zero-slash and previous span. assert_eq!(Staking::force_era(), Forcing::NotForcing); assert!(>::contains_key(11)); assert!(Session::validators().contains(&11)); @@ -2096,7 +2086,7 @@ fn slash_in_old_span_does_not_deselect() { // or non-zero. assert_eq!(Staking::force_era(), Forcing::NotForcing); assert!(>::contains_key(11)); - assert!(Session::validators().contains(&11)); + assert!(Session::validators().contains(&11)); assert_ledger_consistent(11); }); } @@ -2135,7 +2125,7 @@ fn reporters_receive_their_slice() { #[test] fn subsequent_reports_in_same_span_pay_out_less() { // This test verifies that the reporters of the offence receive their slice from the slashed - // amount. + // amount, but less and less if they submit multiple reports in one span. ExtBuilder::default().build().execute_with(|| { // The reporters' reward is calculated from the total exposure. let initial_balance = 1125; @@ -2242,12 +2232,16 @@ fn dont_slash_if_fraction_is_zero() { // The validator hasn't been slashed. The new era is not forced. assert_eq!(Balances::free_balance(11), 1000); + assert_eq!(Staking::force_era(), Forcing::ForceNew); + assert_ledger_consistent(11); }); } #[test] fn only_slash_for_max_in_era() { + // multiple slashes within one era are only applied if it is more than any previous slash in the + // same era. ExtBuilder::default().build().execute_with(|| { assert_eq!(Balances::free_balance(11), 1000); @@ -2296,6 +2290,7 @@ fn only_slash_for_max_in_era() { #[test] fn garbage_collection_after_slashing() { + // ensures that `SlashingSpans` and `SpanSlash` of an account is removed after reaping. ExtBuilder::default().existential_deposit(2).build().execute_with(|| { assert_eq!(Balances::free_balance(11), 256_000); @@ -2338,27 +2333,28 @@ fn garbage_collection_after_slashing() { #[test] fn garbage_collection_on_window_pruning() { + // ensures that `ValidatorSlashInEra` and `NominatorSlashInEra` are cleared after + // `BondingDuration`. ExtBuilder::default().build().execute_with(|| { - start_era(1); + mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); + let now = Staking::active_era().unwrap().index; - let exposure = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); + let exposure = Staking::eras_stakers(now, 11); assert_eq!(Balances::free_balance(101), 2000); let nominated_value = exposure.others.iter().find(|o| o.who == 101).unwrap().value; on_offence_now( &[ OffenceDetails { - offender: (11, Staking::eras_stakers(Staking::active_era().unwrap().index, 11)), + offender: (11, Staking::eras_stakers(now, 11)), reporters: vec![], }, ], &[Perbill::from_percent(10)], ); - let now = Staking::active_era().unwrap().index; - assert_eq!(Balances::free_balance(11), 900); assert_eq!(Balances::free_balance(101), 2000 - (nominated_value / 10)); @@ -2370,7 +2366,7 @@ fn garbage_collection_on_window_pruning() { assert!(::ValidatorSlashInEra::get(&now, &11).is_some()); assert!(::NominatorSlashInEra::get(&now, &101).is_some()); - start_era(era); + mock::start_era(era); } assert!(::ValidatorSlashInEra::get(&now, &11).is_none()); @@ -2381,19 +2377,17 @@ fn garbage_collection_on_window_pruning() { #[test] fn slashing_nominators_by_span_max() { ExtBuilder::default().build().execute_with(|| { - start_era(1); - start_era(2); - start_era(3); + mock::start_era(1); + mock::start_era(2); + mock::start_era(3); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(21), 2000); assert_eq!(Balances::free_balance(101), 2000); assert_eq!(Staking::slashable_balance_of(&21), 1000); - let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, 21); - assert_eq!(Balances::free_balance(101), 2000); let nominated_value_11 = exposure_11.others.iter().find(|o| o.who == 101).unwrap().value; let nominated_value_21 = exposure_21.others.iter().find(|o| o.who == 101).unwrap().value; @@ -2481,9 +2475,9 @@ fn slashing_nominators_by_span_max() { #[test] fn slashes_are_summed_across_spans() { ExtBuilder::default().build().execute_with(|| { - start_era(1); - start_era(2); - start_era(3); + mock::start_era(1); + mock::start_era(2); + mock::start_era(3); assert_eq!(Balances::free_balance(21), 2000); assert_eq!(Staking::slashable_balance_of(&21), 1000); @@ -2511,7 +2505,7 @@ fn slashes_are_summed_across_spans() { // 21 has been force-chilled. re-signal intent to validate. Staking::validate(Origin::signed(20), Default::default()).unwrap(); - start_era(4); + mock::start_era(4); assert_eq!(Staking::slashable_balance_of(&21), 900); @@ -2539,7 +2533,7 @@ fn slashes_are_summed_across_spans() { #[test] fn deferred_slashes_are_deferred() { ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { - start_era(1); + mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2560,19 +2554,19 @@ fn deferred_slashes_are_deferred() { assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); - start_era(2); + mock::start_era(2); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); - start_era(3); + mock::start_era(3); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); // at the start of era 4, slashes from era 1 are processed, // after being deferred for at least 2 full eras. - start_era(4); + mock::start_era(4); assert_eq!(Balances::free_balance(11), 900); assert_eq!(Balances::free_balance(101), 2000 - (nominated_value / 10)); @@ -2582,7 +2576,7 @@ fn deferred_slashes_are_deferred() { #[test] fn remove_deferred() { ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { - start_era(1); + mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2603,7 +2597,7 @@ fn remove_deferred() { assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); - start_era(2); + mock::start_era(2); on_offence_in_era( &[ @@ -2627,20 +2621,20 @@ fn remove_deferred() { assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); - start_era(3); + mock::start_era(3); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); // at the start of era 4, slashes from era 1 are processed, // after being deferred for at least 2 full eras. - start_era(4); + mock::start_era(4); // the first slash for 10% was cancelled, so no effect. assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); - start_era(5); + mock::start_era(5); let slash_10 = Perbill::from_percent(10); let slash_15 = Perbill::from_percent(15); @@ -2658,7 +2652,7 @@ fn remove_deferred() { #[test] fn remove_multi_deferred() { ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { - start_era(1); + mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2737,35 +2731,1101 @@ fn remove_multi_deferred() { let slashes = ::UnappliedSlashes::get(&1); assert_eq!(slashes.len(), 2); - println!("Slashes: {:?}", slashes); assert_eq!(slashes[0].validator, 21); assert_eq!(slashes[1].validator, 42); }) } +mod offchain_phragmen { + use crate::*; + use frame_support::{assert_noop, assert_ok}; + use sp_runtime::transaction_validity::TransactionSource; + use mock::*; + use parking_lot::RwLock; + use sp_core::offchain::{ + testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, + OffchainExt, TransactionPoolExt, + }; + use sp_io::TestExternalities; + use sp_phragmen::StakedAssignment; + use frame_support::traits::OffchainWorker; + use std::sync::Arc; + use substrate_test_utils::assert_eq_uvec; + + fn percent(x: u16) -> OffchainAccuracy { + OffchainAccuracy::from_percent(x) + } + + /// setup a new set of validators and nominator storage items independent of the parent mock + /// file. This produces a edge graph that can be reduced. + fn build_offchain_phragmen_test_ext() { + for i in (10..=40).step_by(10) { + // Note: we respect the convention of the mock (10, 11 pairs etc.) since these accounts + // have corresponding keys in session which makes everything more ergonomic and + // realistic. + bond_validator(i + 1, i, 100); + } + + let mut voter = 1; + bond_nominator(voter, 1000 + voter, 100, vec![11]); + voter = 2; + bond_nominator(voter, 1000 + voter, 100, vec![11, 11]); + voter = 3; + bond_nominator(voter, 1000 + voter, 100, vec![21, 41]); + voter = 4; + bond_nominator(voter, 1000 + voter, 100, vec![21, 31, 41]); + voter = 5; + bond_nominator(voter, 1000 + voter, 100, vec![21, 31, 41]); + } + + fn offchainify(ext: &mut TestExternalities) -> Arc> { + let (offchain, _state) = TestOffchainExt::new(); + let (pool, state) = TestTransactionPoolExt::new(); + + ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); + + state + } + + #[test] + fn is_current_session_final_works() { + ExtBuilder::default() + .session_per_era(3) + .build() + .execute_with(|| { + mock::start_era(1); + assert_eq!(Session::current_index(), 3); + assert_eq!(Staking::current_era(), Some(1)); + assert_eq!(Staking::is_current_session_final(), false); + + start_session(4); + assert_eq!(Session::current_index(), 4); + assert_eq!(Staking::current_era(), Some(1)); + assert_eq!(Staking::is_current_session_final(), true); + + start_session(5); + assert_eq!(Session::current_index(), 5); + // era changed. + assert_eq!(Staking::current_era(), Some(2)); + assert_eq!(Staking::is_current_session_final(), false); + }) + } + + #[test] + fn offchain_election_flag_is_triggered() { + ExtBuilder::default() + .session_per_era(5) + .session_length(10) + .election_lookahead(3) + .build() + .execute_with(|| { + run_to_block(7); + assert_session_era!(0, 0); + + run_to_block(10); + assert_session_era!(1, 0); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert!(Staking::snapshot_nominators().is_none()); + assert!(Staking::snapshot_validators().is_none()); + + run_to_block(36); + assert_session_era!(3, 0); + + // fist era has session 0, which has 0 blocks length, so we have in total 40 blocks + // in the era. + run_to_block(37); + assert_session_era!(3, 0); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(37)); + assert!(Staking::snapshot_nominators().is_some()); + assert!(Staking::snapshot_validators().is_some()); + + run_to_block(38); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(37)); + + run_to_block(39); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(37)); + + run_to_block(40); + assert_session_era!(4, 0); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert!(Staking::snapshot_nominators().is_none()); + assert!(Staking::snapshot_validators().is_none()); + + run_to_block(86); + assert_session_era!(8, 1); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert!(Staking::snapshot_nominators().is_none()); + assert!(Staking::snapshot_validators().is_none()); + + // second era onwards has 50 blocks per era. + run_to_block(87); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(87)); + assert!(Staking::snapshot_nominators().is_some()); + assert!(Staking::snapshot_validators().is_some()); + + run_to_block(90); + assert_session_era!(9, 1); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert!(Staking::snapshot_nominators().is_none()); + assert!(Staking::snapshot_validators().is_none()); + }) + } + + #[test] + fn election_on_chain_fallback_works() { + ExtBuilder::default().build().execute_with(|| { + start_session(1); + start_session(2); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + // some election must have happened by now. + assert_eq!( + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| { + if let MetaEvent::staking(inner) = e { + Some(inner) + } else { + None + } + }) + .last() + .unwrap(), + RawEvent::StakingElection(ElectionCompute::OnChain), + ); + }) + } + + #[test] + #[ignore] // This takes a few mins + fn offchain_wont_work_if_snapshot_fails() { + ExtBuilder::default() + .offchain_phragmen_ext() + .election_lookahead(3) + .build() + .execute_with(|| { + run_to_block(12); + assert!(Staking::snapshot_validators().is_some()); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); + + // validate more than the limit + let limit: NominatorIndex = ValidatorIndex::max_value() as NominatorIndex + 1; + let ctrl = 1_000_000; + for i in 0..limit { + bond_validator((1000 + i).into(), (1000 + i + ctrl).into(), 100); + } + + // window stays closed since no snapshot was taken. + run_to_block(27); + assert!(Staking::snapshot_validators().is_none()); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + }) + } + + #[test] + fn staking_is_locked_when_election_window_open() { + ExtBuilder::default() + .offchain_phragmen_ext() + .election_lookahead(3) + .build() + .execute_with(|| { + run_to_block(12); + assert!(Staking::snapshot_validators().is_some()); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); + + let call = crate::Call::bond(999, 998, Default::default()); + let outer: mock::Call = call.into(); + + let lock_staking: LockStakingStatus = Default::default(); + assert_eq!( + lock_staking.validate(&10, &outer, &Default::default(), Default::default(),), + TransactionValidity::Err(InvalidTransaction::Stale.into()), + ) + }) + } + + #[test] + fn signed_result_can_be_submitted() { + // should check that we have a new validator set normally, + // event says that it comes from offchain. + ExtBuilder::default() + .offchain_phragmen_ext() + .build() + .execute_with(|| { + run_to_block(12); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); + assert!(Staking::snapshot_validators().is_some()); + + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + + let queued_result = Staking::queued_elected().unwrap(); + assert_eq!(queued_result.compute, ElectionCompute::Signed); + + run_to_block(15); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + assert_eq!( + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| { + if let MetaEvent::staking(inner) = e { + Some(inner) + } else { + None + } + }) + .last() + .unwrap(), + RawEvent::StakingElection(ElectionCompute::Signed), + ); + }) + } + + #[test] + fn signed_result_can_be_submitted_later() { + // same as `signed_result_can_be_submitted` but at a later block. + ExtBuilder::default() + .offchain_phragmen_ext() + .build() + .execute_with(|| { + run_to_block(14); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); + + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + + let queued_result = Staking::queued_elected().unwrap(); + assert_eq!(queued_result.compute, ElectionCompute::Signed); + + run_to_block(15); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + assert_eq!( + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| { + if let MetaEvent::staking(inner) = e { + Some(inner) + } else { + None + } + }) + .last() + .unwrap(), + RawEvent::StakingElection(ElectionCompute::Signed), + ); + }) + } + + #[test] + fn early_solution_submission_is_rejected() { + // should check that we have a new validator set normally, + // event says that it comes from offchain. + ExtBuilder::default() + .offchain_phragmen_ext() + .build() + .execute_with(|| { + run_to_block(11); + // submission is not yet allowed + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + // create all the indices just to build the solution. + Staking::create_stakers_snapshot(); + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + Staking::kill_stakers_snapshot(); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenEarlySubmission, + ); + }) + } + + #[test] + fn weak_solution_is_rejected() { + // A solution which is weaker than what we currently have on-chain is rejected. + ExtBuilder::default() + .offchain_phragmen_ext() + .has_stakers(false) + .validator_count(4) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + // a good solution + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + + // a bad solution + let (compact, winners, score) = horrible_phragmen_with_post_processing(false); + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenWeakSubmission, + ); + }) + } + + #[test] + fn better_solution_is_accepted() { + // A solution which is better than what we currently have on-chain is accepted. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + // a meeeeh solution + let (compact, winners, score) = horrible_phragmen_with_post_processing(false); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + + // a better solution + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + }) + } + + #[test] + fn offchain_worker_runs_when_window_open() { + // at the end of the first finalized block with ElectionStatus::open(_), it should execute. + let mut ext = ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(2) + .build(); + let state = offchainify(&mut ext); + ext.execute_with(|| { + run_to_block(12); + + // local key 11 is in the elected set. + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(state.read().transactions.len(), 0); + Staking::offchain_worker(12); + assert_eq!(state.read().transactions.len(), 1); + + let encoded = state.read().transactions[0].clone(); + let extrinsic: Extrinsic = Decode::decode(&mut &*encoded).unwrap(); + + let call = extrinsic.call; + let inner = match call { + mock::Call::Staking(inner) => inner, + }; + + assert_eq!( + ::validate_unsigned( + TransactionSource::Local, + &inner, + ), + TransactionValidity::Ok(ValidTransaction { + priority: (1 << 20) + 1125, // the proposed slot stake. + requires: vec![], + provides: vec![("StakingOffchain", active_era()).encode()], + longevity: 3, + propagate: false, + }) + ) + }) + } + + #[test] + fn mediocre_submission_from_authority_is_early_rejected() { + let mut ext = ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .build(); + let state = offchainify(&mut ext); + ext.execute_with(|| { + run_to_block(12); + // put a good solution on-chain + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ),); + + // now run the offchain worker in the same chain state. + Staking::offchain_worker(12); + assert_eq!(state.read().transactions.len(), 1); + + let encoded = state.read().transactions[0].clone(); + let extrinsic: Extrinsic = Decode::decode(&mut &*encoded).unwrap(); + + let call = extrinsic.call; + let inner = match call { + mock::Call::Staking(inner) => inner, + }; + + // pass this call to ValidateUnsigned + assert_eq!( + ::validate_unsigned( + TransactionSource::Local, + &inner, + ), + TransactionValidity::Err( + InvalidTransaction::Custom(>::PhragmenWeakSubmission.as_u8()).into(), + ), + ) + }) + } + + #[test] + fn invalid_phragmen_result_correct_number_of_winners() { + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + ValidatorCount::put(3); + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + ValidatorCount::put(4); + + assert_eq!(winners.len(), 3); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusWinnerCount, + ); + }) + } + + #[test] + fn invalid_phragmen_result_correct_number_of_winners_1() { + // if we have too little validators, then the number of candidates is the bound. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(8) // we simply cannot elect 8 + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + ValidatorCount::put(3); + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + ValidatorCount::put(4); + + assert_eq!(winners.len(), 3); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusWinnerCount, + ); + }) + } + + #[test] + fn invalid_phragmen_result_correct_number_of_winners_2() { + // if we have too little validators, then the number of candidates is the bound. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(8) // we simply cannot elect 8 + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + let (compact, winners, score) = prepare_submission_with(true, |_| {}); + + assert_eq!(winners.len(), 4); + + // all good. We chose 4 and it works. + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ),); + }) + } + + #[test] + fn invalid_phragmen_result_out_of_bound_nominator_index() { + // A nominator index which is simply invalid + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4); + assert_eq!(Staking::snapshot_validators().unwrap().len(), 4); + let (mut compact, winners, score) = prepare_submission_with(true, |_| {}); + + // index 9 doesn't exist. + compact.votes1.push((9, 2)); + + // The error type sadly cannot be more specific now. + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusCompact, + ); + }) + } + + #[test] + fn invalid_phragmen_result_out_of_bound_validator_index() { + // A validator index which is out of bound + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4); + assert_eq!(Staking::snapshot_validators().unwrap().len(), 4); + let (mut compact, winners, score) = prepare_submission_with(true, |_| {}); + + // index 4 doesn't exist. + compact.votes1.push((3, 4)); + + // The error type sadly cannot be more specific now. + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusCompact, + ); + }) + } + + #[test] + fn invalid_phragmen_result_out_of_bound_winner_index() { + // A winner index which is simply invalid + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4); + assert_eq!(Staking::snapshot_validators().unwrap().len(), 4); + let (compact, _, score) = prepare_submission_with(true, |_| {}); + + // index 4 doesn't exist. + let winners = vec![0, 1, 2, 4]; + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusWinner, + ); + }) + } + + #[test] + fn invalid_phragmen_result_non_winner_validator_index() { + // An edge that points to a correct validator index who is NOT a winner. This is very + // similar to the test that raises `PhragmenBogusNomination`. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(2) // we select only 2. + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + assert_eq!(Staking::snapshot_nominators().unwrap().len(), 5 + 4); + assert_eq!(Staking::snapshot_validators().unwrap().len(), 4); + let (compact, winners, score) = prepare_submission_with(true, |a| { + a.iter_mut() + .find(|x| x.who == 5) + // all 3 cannot be among the winners. Although, all of them are validator + // candidates. + .map(|x| x.distribution = vec![(21, 50), (41, 30), (31, 20)]); + }); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusEdge, + ); + }) + } + + #[test] + fn invalid_phragmen_result_wrong_self_vote() { + // A self vote for someone else. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + let (compact, winners, score) = prepare_submission_with(true, |a| { + // mutate a self vote to target someone else. That someone else is still among the + // winners + a.iter_mut().find(|x| x.who == 11).map(|x| { + x.distribution + .iter_mut() + .find(|y| y.0 == 11) + .map(|y| y.0 = 21) + }); + }); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusSelfVote, + ); + }) + } + + #[test] + fn invalid_phragmen_result_wrong_self_vote_2() { + // A self validator voting for someone else next to self vote. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + let (compact, winners, score) = prepare_submission_with(true, |a| { + // Remove the self vote. + a.retain(|x| x.who != 11); + // add is as a new double vote + a.push(StakedAssignment { + who: 11, + distribution: vec![(11, 50), (21, 50)], + }); + }); + + // This raises score issue. + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusSelfVote, + ); + }) + } + + #[test] + fn invalid_phragmen_result_over_stake() { + // Someone's edge ratios sums to more than 100%. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + // Note: we don't reduce here to be able to tweak votes3. votes3 will vanish if you + // reduce. + let (mut compact, winners, score) = prepare_submission_with(false, |_| {}); + + if let Some(c) = compact.votes3.iter_mut().find(|x| x.0 == 0) { + // by default it should have been (0, [(2, 33%), (1, 33%)], 0) + // now the sum is above 100% + c.1 = [(2, percent(66)), (1, percent(66))]; + } + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusCompact, + ); + }) + } + + #[test] + fn invalid_phragmen_result_under_stake() { + // at the time of this writing, we cannot under stake someone. The compact assignment works + // in a way that some of the stakes are presented by the submitter, and the last one is read + // from chain by subtracting the rest from total. Hence, the sum is always correct. + // This test is only here as a demonstration. + } + + #[test] + fn invalid_phragmen_result_invalid_target_stealing() { + // A valid voter who voted for someone who is a candidate, and is a correct winner, but is + // actually NOT nominated by this nominator. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + let (compact, winners, score) = prepare_submission_with(false, |a| { + // 3 only voted for 20 and 40. We add a fake vote to 30. The stake sum is still + // correctly 100. + a.iter_mut() + .find(|x| x.who == 3) + .map(|x| x.distribution = vec![(21, 50), (41, 30), (31, 20)]); + }); + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusNomination, + ); + }) + } + + #[test] + fn nomination_slash_filter_is_checked() { + // If a nominator has voted for someone who has been recently slashed, that particular + // nomination should be disabled for the upcoming election. A solution must respect this + // rule. + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + + // finalize the round with fallback. This is needed since all nominator submission + // are in era zero and we want this one to pass with no problems. + run_to_block(15); + + // go to the next session to trigger mock::start_era and bump the active era + run_to_block(20); + + // slash 10. This must happen outside of the election window. + let offender_expo = Staking::eras_stakers(active_era(), 11); + on_offence_now( + &[OffenceDetails { + offender: (11, offender_expo.clone()), + reporters: vec![], + }], + &[Perbill::from_percent(50)], + ); + + // validate 10 again for the next round. But this guy will not have the votes that + // it should have had from 1 and 2. + assert_ok!(Staking::validate( + Origin::signed(10), + Default::default() + )); + + // open the election window and create snapshots. + run_to_block(32); + + // a solution that has been prepared after the slash. + let (compact, winners, score) = prepare_submission_with(false, |a| { + // no one is allowed to vote for 10, except for itself. + a.into_iter() + .filter(|s| s.who != 11) + .for_each(|s| + assert!(s.distribution.iter().find(|(t, _)| *t == 11).is_none()) + ); + }); + + // can be submitted. + assert_ok!(Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + )); + + // a wrong solution. + let (compact, winners, score) = prepare_submission_with(false, |a| { + // add back the vote that has been filtered out. + a.push(StakedAssignment { + who: 1, + distribution: vec![(11, 100)] + }); + }); + + // is rejected. + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenSlashedNomination, + ); + }) + } + + #[test] + fn invalid_phragmen_result_wrong_score() { + // A valid voter who's total distributed stake is more than what they bond + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + build_offchain_phragmen_test_ext(); + run_to_block(12); + + let (compact, winners, mut score) = prepare_submission_with(true, |_| {}); + score[0] += 1; + + assert_noop!( + Staking::submit_election_solution( + Origin::signed(10), + winners, + compact, + score, + active_era(), + ), + Error::::PhragmenBogusScore, + ); + }) + } + + #[test] + fn offchain_storage_is_set() { + let mut ext = ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .build(); + let state = offchainify(&mut ext); + + ext.execute_with(|| { + use offchain_election::OFFCHAIN_HEAD_DB; + use sp_runtime::offchain::storage::StorageValueRef; + + run_to_block(12); + + Staking::offchain_worker(12); + // it works + assert_eq!(state.read().transactions.len(), 1); + + // and it is set + let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); + assert_eq!(storage.get::().unwrap().unwrap(), 12); + }) + } + + #[test] + fn offchain_storage_prevents_duplicate() { + let mut ext = ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .build(); + let _ = offchainify(&mut ext); + + ext.execute_with(|| { + use offchain_election::OFFCHAIN_HEAD_DB; + use sp_runtime::offchain::storage::StorageValueRef; + let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB); + + run_to_block(12); + + // first run -- ok + assert_eq!( + offchain_election::set_check_offchain_execution_status::(12), + Ok(()), + ); + assert_eq!(storage.get::().unwrap().unwrap(), 12); + + // re-execute after the next. not allowed. + assert_eq!( + offchain_election::set_check_offchain_execution_status::(13), + Err("recently executed."), + ); + + // a fork like situation -- re-execute 10, 11, 12. But it won't go through. + assert_eq!( + offchain_election::set_check_offchain_execution_status::(10), + Err("fork."), + ); + assert_eq!( + offchain_election::set_check_offchain_execution_status::(11), + Err("fork."), + ); + assert_eq!( + offchain_election::set_check_offchain_execution_status::(12), + Err("recently executed."), + ); + }) + } + + #[test] + #[should_panic] + fn offence_is_blocked_when_window_open() { + ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(4) + .has_stakers(false) + .build() + .execute_with(|| { + run_to_block(12); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); + + let offender_expo = Staking::eras_stakers(active_era(), 10); + + // panic from the impl in mock + on_offence_now( + &[OffenceDetails { + offender: (10, offender_expo.clone()), + reporters: vec![], + }], + &[Perbill::from_percent(10)], + ); + }) + } +} + #[test] -fn slash_kicks_validators_not_nominators() { +fn slash_kicks_validators_not_nominators_and_disables_nominator_for_kicked_validator() { ExtBuilder::default().build().execute_with(|| { - start_era(1); + mock::start_era(1); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + // pre-slash balance assert_eq!(Balances::free_balance(11), 1000); - - let exposure = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); assert_eq!(Balances::free_balance(101), 2000); - let nominated_value = exposure.others.iter().find(|o| o.who == 101).unwrap().value; + + // 11 and 21 both have the support of 100 + let exposure_11 = Staking::eras_stakers(active_era(), &11); + let exposure_21 = Staking::eras_stakers(active_era(), &21); + + assert_eq!(exposure_11.total, 1000 + 125); + assert_eq!(exposure_21.total, 1000 + 375); on_offence_now( - &[ - OffenceDetails { - offender: (11, exposure.clone()), - reporters: vec![], - }, - ], + &[OffenceDetails { + offender: (11, exposure_11.clone()), + reporters: vec![], + }], &[Perbill::from_percent(10)], ); + // post-slash balance + let nominator_slash_amount_11 = 125 / 10; assert_eq!(Balances::free_balance(11), 900); - assert_eq!(Balances::free_balance(101), 2000 - (nominated_value / 10)); + assert_eq!( + Balances::free_balance(101), + 2000 - nominator_slash_amount_11 + ); // This is the best way to check that the validator was chilled; `get` will // return default value. @@ -2777,8 +3837,23 @@ fn slash_kicks_validators_not_nominators() { // and make sure that the vote will be ignored even if the validator // re-registers. - let last_slash = ::SlashingSpans::get(&11).unwrap().last_nonzero_slash(); + let last_slash = ::SlashingSpans::get(&11) + .unwrap() + .last_nonzero_slash(); assert!(nominations.submitted_in < last_slash); + + // actually re-bond the slashed validator + assert_ok!(Staking::validate(Origin::signed(10), Default::default())); + + mock::start_era(2); + let exposure_11 = Staking::eras_stakers(active_era(), &11); + let exposure_21 = Staking::eras_stakers(active_era(), &21); + + // 10 is re-elected, but without the support of 100 + assert_eq!(exposure_11.total, 900); + + // 20 is re-elected, with the (almost) entire support of 100 + assert_eq!(exposure_21.total, 1000 + 500 - nominator_slash_amount_11); }); } @@ -2804,7 +3879,7 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { let total_payout_0 = current_total_payout_for_duration(3000); assert!(total_payout_0 > 10); // Test is meaningful if reward something - start_era(1); + mock::start_era(1); >::reward_by_ids(vec![(11, 1)]); // Change total issuance in order to modify total payout @@ -2814,7 +3889,7 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { assert!(total_payout_1 > 10); // Test is meaningful if reward something assert!(total_payout_1 != total_payout_0); - start_era(2); + mock::start_era(2); >::reward_by_ids(vec![(11, 1)]); // Change total issuance in order to modify total payout @@ -2825,7 +3900,7 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { assert!(total_payout_2 != total_payout_0); assert!(total_payout_2 != total_payout_1); - start_era(Staking::history_depth() + 1); + mock::start_era(Staking::history_depth() + 1); let active_era = Staking::active_era().unwrap().index; @@ -2835,37 +3910,19 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { // Last kept is 1: assert!(current_era - Staking::history_depth() == 1); assert_noop!( - Staking::payout_validator(Origin::signed(10), 0), - // Fail: Era out of history - Error::::InvalidEraToReward - ); - assert_ok!(Staking::payout_validator(Origin::signed(10), 1)); - assert_ok!(Staking::payout_validator(Origin::signed(10), 2)); - assert_noop!( - Staking::payout_validator(Origin::signed(10), 2), - // Fail: Double claim - Error::::InvalidEraToReward - ); - assert_noop!( - Staking::payout_validator(Origin::signed(10), active_era), - // Fail: Era not finished yet - Error::::InvalidEraToReward - ); - - assert_noop!( - Staking::payout_nominator(Origin::signed(100), 0, vec![(11, 0)]), + Staking::payout_stakers(Origin::signed(1337), 11, 0), // Fail: Era out of history Error::::InvalidEraToReward ); - assert_ok!(Staking::payout_nominator(Origin::signed(100), 1, vec![(11, 0)])); - assert_ok!(Staking::payout_nominator(Origin::signed(100), 2, vec![(11, 0)])); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 1)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 2)); assert_noop!( - Staking::payout_nominator(Origin::signed(100), 2, vec![(11, 0)]), + Staking::payout_stakers(Origin::signed(1337), 11, 2), // Fail: Double claim - Error::::InvalidEraToReward + Error::::AlreadyClaimed ); assert_noop!( - Staking::payout_nominator(Origin::signed(100), active_era, vec![(11, 0)]), + Staking::payout_stakers(Origin::signed(1337), 11, active_era), // Fail: Era not finished yet Error::::InvalidEraToReward ); @@ -2887,11 +3944,11 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { #[test] fn zero_slash_keeps_nominators() { ExtBuilder::default().build().execute_with(|| { - start_era(1); + mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); - let exposure = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); + let exposure = Staking::eras_stakers(active_era(), 11); assert_eq!(Balances::free_balance(101), 2000); on_offence_now( @@ -3004,10 +4061,6 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( mock::start_era(2); mock::make_all_reward_payment(1); - // nominator 10_000 can't get its reward because exposure is clipped. However it will try - // to query other people reward. - assert_ok!(Staking::payout_nominator(Origin::signed(20_000), 1, vec![(11, 0)])); - // Assert only nominators from 1 to Max are rewarded for i in 0..=::MaxNominatorRewardedPerValidator::get() { let stash = 10_000 + i as u64; @@ -3024,7 +4077,7 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( #[test] fn set_history_depth_works() { ExtBuilder::default().build().execute_with(|| { - start_era(10); + mock::start_era(10); Staking::set_history_depth(Origin::ROOT, 20).unwrap(); assert!(::ErasTotalStake::contains_key(10 - 4)); assert!(::ErasTotalStake::contains_key(10 - 5)); @@ -3039,3 +4092,630 @@ fn set_history_depth_works() { assert!(!::ErasTotalStake::contains_key(10 - 5)); }); } + +#[test] +fn test_payout_stakers() { + // Here we will test validator can set `max_nominators_payout` and it works. + // We also test that `payout_extra_nominators` works. + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + let balance = 1000; + // Create three validators: + bond_validator(11, 10, balance); // Default(64) + + // Create nominators, targeting stash of validators + for i in 0..100 { + bond_nominator(1000 + i, 100 + i, balance + i, vec![11]); + } + + mock::start_era(1); + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(2); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 1)); + + // Top 64 nominators of validator 11 automatically paid out, including the validator + // Validator payout goes to controller. + assert!(Balances::free_balance(&10) > balance); + for i in 36..100 { + assert!(Balances::free_balance(&(100 + i)) > balance + i); + } + // The bottom 36 do not + for i in 0..36 { + assert_eq!(Balances::free_balance(&(100 + i)), balance + i); + } + + // We track rewards in `claimed_rewards` vec + assert_eq!( + Staking::ledger(&10), + Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: vec![1] }) + ); + + for i in 3..16 { + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(i); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, i - 1)); + } + + // We track rewards in `claimed_rewards` vec + assert_eq!( + Staking::ledger(&10), + Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: (1..=14).collect() }) + ); + + for i in 16..100 { + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(i); + } + + // We clean it up as history passes + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 15)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 98)); + assert_eq!( + Staking::ledger(&10), + Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: vec![15, 98] }) + ); + + // Out of order claims works. + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 69)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 23)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 42)); + assert_eq!( + Staking::ledger(&10), + Some(StakingLedger { stash: 11, total: 1000, active: 1000, unlocking: vec![], claimed_rewards: vec![15, 23, 42, 69, 98] }) + ); + }); +} + +#[test] +fn payout_stakers_handles_basic_errors() { + // Here we will test payouts handle all errors. + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + // Same setup as the test above + let balance = 1000; + bond_validator(11, 10, balance); // Default(64) + + // Create nominators, targeting stash + for i in 0..100 { + bond_nominator(1000 + i, 100 + i, balance + i, vec![11]); + } + + mock::start_era(1); + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(2); + + // Wrong Era, too big + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 11, 2), Error::::InvalidEraToReward); + // Wrong Staker + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 10, 1), Error::::NotStash); + + for i in 3..100 { + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(i); + } + // We are at era 99, with history depth of 84 + // We should be able to payout era 15 through 98 (84 total eras), but not 14 or 99. + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 11, 14), Error::::InvalidEraToReward); + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 11, 99), Error::::InvalidEraToReward); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 15)); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 98)); + + // Can't claim again + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 11, 15), Error::::AlreadyClaimed); + assert_noop!(Staking::payout_stakers(Origin::signed(1337), 11, 98), Error::::AlreadyClaimed); + }); +} + +#[test] +fn bond_during_era_correctly_populates_claimed_rewards() { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + // Era = None + bond_validator(9, 8, 1000); + assert_eq!( + Staking::ledger(&8), + Some(StakingLedger { + stash: 9, + total: 1000, + active: 1000, + unlocking: vec![], + claimed_rewards: vec![], + }) + ); + mock::start_era(5); + bond_validator(11, 10, 1000); + assert_eq!( + Staking::ledger(&10), + Some(StakingLedger { + stash: 11, + total: 1000, + active: 1000, + unlocking: vec![], + claimed_rewards: (0..5).collect(), + }) + ); + mock::start_era(99); + bond_validator(13, 12, 1000); + assert_eq!( + Staking::ledger(&12), + Some(StakingLedger { + stash: 13, + total: 1000, + active: 1000, + unlocking: vec![], + claimed_rewards: (15..99).collect(), + }) + ); + }); +} + +/* These migration tests below can be removed once migration code is removed */ + +#[test] +fn assert_migration_is_noop() { + let kusama_active_era = "4a0200000190e2721171010000"; + let era = ActiveEraInfo::decode(&mut &hex::decode(kusama_active_era).unwrap()[..]).unwrap(); + assert_eq!(era.index, 586); + assert_eq!(era.start, Some(1585135674000)); +} + +#[test] +fn test_last_reward_migration() { + use sp_storage::Storage; + + let mut s = Storage::default(); + + #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] + struct OldStakingLedger { + pub stash: AccountId, + #[codec(compact)] + pub total: Balance, + #[codec(compact)] + pub active: Balance, + pub unlocking: Vec>, + pub last_reward: Option, + } + + let old_staking10 = OldStakingLedger:: { + stash: 0, + total: 10, + active: 10, + unlocking: vec![UnlockChunk{ value: 1234, era: 56}], + last_reward: Some(8), + }; + + let old_staking11 = OldStakingLedger:: { + stash: 1, + total: 0, + active: 0, + unlocking: vec![], + last_reward: None, + }; + + let old_staking12 = OldStakingLedger:: { + stash: 2, + total: 100, + active: 100, + unlocking: vec![UnlockChunk{ value: 9876, era: 54}, UnlockChunk{ value: 98, era: 76}], + last_reward: Some(23), + }; + + let old_staking13 = OldStakingLedger:: { + stash: 3, + total: 100, + active: 100, + unlocking: vec![], + last_reward: Some(23), + }; + + let data = vec![ + ( + Ledger::::hashed_key_for(10), + old_staking10.encode().to_vec() + ), + ( + Ledger::::hashed_key_for(11), + old_staking11.encode().to_vec() + ), + ( + Ledger::::hashed_key_for(12), + old_staking12.encode().to_vec() + ), + ( + Ledger::::hashed_key_for(13), + old_staking13.encode().to_vec() + ), + ]; + + s.top = data.into_iter().collect(); + sp_io::TestExternalities::new(s).execute_with(|| { + HistoryDepth::put(84); + CurrentEra::put(99); + let nominations = Nominations:: { + targets: vec![], + submitted_in: 0, + suppressed: false + }; + Nominators::::insert(3, nominations); + Bonded::::insert(3, 13); + Staking::migrate_last_reward_to_claimed_rewards(); + // Test staker out of range + assert_eq!( + Ledger::::get(10), + Some(StakingLedger { + stash: 0, + total: 10, + active: 10, + unlocking: vec![UnlockChunk{ value: 1234, era: 56}], + claimed_rewards: vec![], + }) + ); + // Test staker none + assert_eq!( + Ledger::::get(11), + Some(StakingLedger { + stash: 1, + total: 0, + active: 0, + unlocking: vec![], + claimed_rewards: vec![], + }) + ); + // Test staker migration + assert_eq!( + Ledger::::get(12), + Some(StakingLedger { + stash: 2, + total: 100, + active: 100, + unlocking: vec![UnlockChunk{ value: 9876, era: 54}, UnlockChunk{ value: 98, era: 76}], + claimed_rewards: vec![15,16,17,18,19,20,21,22,23], + }) + ); + // Test nominator migration + assert_eq!( + Ledger::::get(13), + Some(StakingLedger { + stash: 3, + total: 100, + active: 100, + unlocking: vec![], + claimed_rewards: vec![15,16,17,18,19,20,21,22,23], + }) + ); + }); +} + +#[test] +fn rewards_should_work_before_migration() { + // should check that before migration: + // * rewards get recorded per session + // * rewards get paid per Era + // * Check that nominators are also rewarded + ExtBuilder::default().nominate(true).build().execute_with(|| { + MigrateEra::put(10); + let init_balance_10 = Balances::total_balance(&10); + let init_balance_11 = Balances::total_balance(&11); + let init_balance_20 = Balances::total_balance(&20); + let init_balance_21 = Balances::total_balance(&21); + let init_balance_100 = Balances::total_balance(&100); + let init_balance_101 = Balances::total_balance(&101); + + // Check state + Payee::::insert(11, RewardDestination::Controller); + Payee::::insert(21, RewardDestination::Controller); + Payee::::insert(101, RewardDestination::Controller); + + >::reward_by_ids(vec![(11, 50)]); + >::reward_by_ids(vec![(11, 50)]); + // This is the second validator of the current elected set. + >::reward_by_ids(vec![(21, 50)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 10); // Test is meaningful if reward something + + start_session(1); + + assert_eq!(Balances::total_balance(&10), init_balance_10); + assert_eq!(Balances::total_balance(&11), init_balance_11); + assert_eq!(Balances::total_balance(&20), init_balance_20); + assert_eq!(Balances::total_balance(&21), init_balance_21); + assert_eq!(Balances::total_balance(&100), init_balance_100); + assert_eq!(Balances::total_balance(&101), init_balance_101); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { + total: 50*3, + individual: vec![(11, 100), (21, 50)].into_iter().collect(), + }); + let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); + let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); + let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); + let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); + + start_session(2); + start_session(3); + + assert_eq!(Staking::active_era().unwrap().index, 1); + mock::make_all_reward_payment_before_migration(0); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2/3 + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + + assert_eq_uvec!(Session::validators(), vec![11, 21]); + >::reward_by_ids(vec![(11, 1)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_1 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_1 > 10); // Test is meaningful if reward something + + mock::start_era(2); + mock::make_all_reward_payment_before_migration(1); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + }); +} + +#[test] +fn migrate_era_should_work() { + // should check that before and after migration: + // * rewards get recorded per session + // * rewards get paid per Era + // * Check that nominators are also rewarded + ExtBuilder::default().nominate(true).build().execute_with(|| { + MigrateEra::put(1); + let init_balance_10 = Balances::total_balance(&10); + let init_balance_11 = Balances::total_balance(&11); + let init_balance_20 = Balances::total_balance(&20); + let init_balance_21 = Balances::total_balance(&21); + let init_balance_100 = Balances::total_balance(&100); + let init_balance_101 = Balances::total_balance(&101); + + // Check state + Payee::::insert(11, RewardDestination::Controller); + Payee::::insert(21, RewardDestination::Controller); + Payee::::insert(101, RewardDestination::Controller); + + >::reward_by_ids(vec![(11, 50)]); + >::reward_by_ids(vec![(11, 50)]); + // This is the second validator of the current elected set. + >::reward_by_ids(vec![(21, 50)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 10); // Test is meaningful if reward something + + start_session(1); + + assert_eq!(Balances::total_balance(&10), init_balance_10); + assert_eq!(Balances::total_balance(&11), init_balance_11); + assert_eq!(Balances::total_balance(&20), init_balance_20); + assert_eq!(Balances::total_balance(&21), init_balance_21); + assert_eq!(Balances::total_balance(&100), init_balance_100); + assert_eq!(Balances::total_balance(&101), init_balance_101); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { + total: 50*3, + individual: vec![(11, 100), (21, 50)].into_iter().collect(), + }); + let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); + let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); + let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); + let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); + + start_session(2); + start_session(3); + + assert_eq!(Staking::active_era().unwrap().index, 1); + mock::make_all_reward_payment_before_migration(0); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2/3 + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + + assert_eq_uvec!(Session::validators(), vec![11, 21]); + >::reward_by_ids(vec![(11, 1)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_1 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_1 > 10); // Test is meaningful if reward something + + mock::start_era(2); + mock::make_all_reward_payment(1); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + }); +} + +#[test] +#[should_panic] +fn migrate_era_should_handle_error() { + ExtBuilder::default().nominate(true).build().execute_with(|| { + MigrateEra::put(1); + let init_balance_10 = Balances::total_balance(&10); + let init_balance_11 = Balances::total_balance(&11); + let init_balance_20 = Balances::total_balance(&20); + let init_balance_21 = Balances::total_balance(&21); + let init_balance_100 = Balances::total_balance(&100); + let init_balance_101 = Balances::total_balance(&101); + + // Check state + Payee::::insert(11, RewardDestination::Controller); + Payee::::insert(21, RewardDestination::Controller); + Payee::::insert(101, RewardDestination::Controller); + + >::reward_by_ids(vec![(11, 50)]); + >::reward_by_ids(vec![(11, 50)]); + // This is the second validator of the current elected set. + >::reward_by_ids(vec![(21, 50)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 10); // Test is meaningful if reward something + + start_session(1); + + assert_eq!(Balances::total_balance(&10), init_balance_10); + assert_eq!(Balances::total_balance(&11), init_balance_11); + assert_eq!(Balances::total_balance(&20), init_balance_20); + assert_eq!(Balances::total_balance(&21), init_balance_21); + assert_eq!(Balances::total_balance(&100), init_balance_100); + assert_eq!(Balances::total_balance(&101), init_balance_101); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { + total: 50*3, + individual: vec![(11, 100), (21, 50)].into_iter().collect(), + }); + + start_session(2); + start_session(3); + + assert_eq!(Staking::active_era().unwrap().index, 1); + mock::make_all_reward_payment(0); + }); +} + +#[test] +#[should_panic] +fn migrate_era_should_handle_errors_2() { + // should check that before and after migration: + // * rewards get recorded per session + // * rewards get paid per Era + // * Check that nominators are also rewarded + ExtBuilder::default().nominate(true).build().execute_with(|| { + MigrateEra::put(1); + let init_balance_10 = Balances::total_balance(&10); + let init_balance_11 = Balances::total_balance(&11); + let init_balance_20 = Balances::total_balance(&20); + let init_balance_21 = Balances::total_balance(&21); + let init_balance_100 = Balances::total_balance(&100); + let init_balance_101 = Balances::total_balance(&101); + + // Check state + Payee::::insert(11, RewardDestination::Controller); + Payee::::insert(21, RewardDestination::Controller); + Payee::::insert(101, RewardDestination::Controller); + + >::reward_by_ids(vec![(11, 50)]); + >::reward_by_ids(vec![(11, 50)]); + // This is the second validator of the current elected set. + >::reward_by_ids(vec![(21, 50)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 10); // Test is meaningful if reward something + + start_session(1); + + assert_eq!(Balances::total_balance(&10), init_balance_10); + assert_eq!(Balances::total_balance(&11), init_balance_11); + assert_eq!(Balances::total_balance(&20), init_balance_20); + assert_eq!(Balances::total_balance(&21), init_balance_21); + assert_eq!(Balances::total_balance(&100), init_balance_100); + assert_eq!(Balances::total_balance(&101), init_balance_101); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { + total: 50*3, + individual: vec![(11, 100), (21, 50)].into_iter().collect(), + }); + let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); + let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); + let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); + let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); + + start_session(2); + start_session(3); + + assert_eq!(Staking::active_era().unwrap().index, 1); + mock::make_all_reward_payment_before_migration(0); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * total_payout_0 * 2/3 + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + + assert_eq_uvec!(Session::validators(), vec![11, 21]); + >::reward_by_ids(vec![(11, 1)]); + + // Compute total payout now for whole duration as other parameter won't change + let total_payout_1 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_1 > 10); // Test is meaningful if reward something + + mock::start_era(2); + mock::make_all_reward_payment_before_migration(1); + + assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); + assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); + assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); + assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); + assert_eq_error_rate!( + Balances::total_balance(&100), + init_balance_100 + + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) + + part_for_100_from_20 * total_payout_0 * 1/3, + 2 + ); + assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + }); +} diff --git a/frame/sudo/Cargo.toml b/frame/sudo/Cargo.toml index bb2bb090026ca59ebd6e37b7eac0fdfadbf7fcf3..4216b94ec12f12ccf8514698a78a25e3e9752ac6 100644 --- a/frame/sudo/Cargo.toml +++ b/frame/sudo/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for sudo" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -31,3 +31,6 @@ std = [ "frame-support/std", "frame-system/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/sudo/src/lib.rs b/frame/sudo/src/lib.rs index 8ee09ba223ab1a393fadb9ff41056dbac21c756b..3f2eacdbf8139a83530fcf7536b6ad981e6af58f 100644 --- a/frame/sudo/src/lib.rs +++ b/frame/sudo/src/lib.rs @@ -58,6 +58,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] //! pub fn privileged_function(origin) -> dispatch::DispatchResult { //! ensure_root(origin)?; //! @@ -86,7 +87,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::prelude::*; -use sp_runtime::{traits::{StaticLookup, Dispatchable}, DispatchError}; +use sp_runtime::traits::{StaticLookup, Dispatchable}; use frame_support::{ Parameter, decl_module, decl_event, decl_storage, decl_error, ensure, @@ -120,7 +121,7 @@ decl_module! { /// - Weight of derivative `call` execution + 10,000. /// # #[weight = FunctionOf( - |args: (&Box<::Call>,)| args.0.get_dispatch_info().weight + 10_000, + |args: (&Box<::Call>,)| args.0.get_dispatch_info().weight + 10_000, |args: (&Box<::Call>,)| args.0.get_dispatch_info().class, true )] @@ -132,7 +133,6 @@ decl_module! { let res = match call.dispatch(frame_system::RawOrigin::Root.into()) { Ok(_) => true, Err(e) => { - let e: DispatchError = e.into(); sp_runtime::print(e); false } @@ -150,6 +150,7 @@ decl_module! { /// - Limited storage reads. /// - One DB change. /// # + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn set_key(origin, new: ::Source) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; @@ -174,7 +175,7 @@ decl_module! { #[weight = FunctionOf( |args: (&::Source, &Box<::Call>,)| { args.1.get_dispatch_info().weight + 10_000 - }, + }, |args: (&::Source, &Box<::Call>,)| { args.1.get_dispatch_info().class }, @@ -190,7 +191,6 @@ decl_module! { let res = match call.dispatch(frame_system::RawOrigin::Signed(who).into()) { Ok(_) => true, Err(e) => { - let e: DispatchError = e.into(); sp_runtime::print(e); false } diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index fc7fbb483562627cfe1cba7bbb7da27a3d58f100..3bad72a115716b6f5c1f6297bf62ef40d24674a2 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -11,10 +11,10 @@ description = "Support code for the runtime." [dependencies] log = "0.4" serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } frame-metadata = { version = "11.0.0-alpha.5", default-features = false, path = "../metadata" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} +sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-arithmetic = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/arithmetic" } @@ -50,3 +50,6 @@ std = [ nightly = [] strict = [] runtime-benchmarks = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index 8befc50cd7caff723639c4f864b0675665ead43f..2f7450e4b86d3ea960a9b9b2e1e3b2c32b1af306 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -16,3 +16,6 @@ frame-support-procedural-tools = { version = "2.0.0-alpha.5", path = "./tools" } proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/src/construct_runtime/mod.rs b/frame/support/procedural/src/construct_runtime/mod.rs index 942a47a533e5f894425d9eb60def32c51ed73ff6..b74a27e7ba936c982512438fc47d5ab7dd5dc379 100644 --- a/frame/support/procedural/src/construct_runtime/mod.rs +++ b/frame/support/procedural/src/construct_runtime/mod.rs @@ -89,7 +89,7 @@ fn construct_runtime_parsed(definition: RuntimeDefinition) -> Result Result( diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index 09afcb9a60d78ae263c162ac011819c746ff9c20..eeeca150d9b9b8419922f73f1ef51ed3ce46e095 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -66,6 +66,7 @@ fn decl_genesis_config_and_impl_default( let genesis_where_clause = &genesis_config.genesis_where_clause; quote!( + /// Genesis config for the module, allow to build genesis storage. #[derive(#scrate::Serialize, #scrate::Deserialize)] #[cfg(feature = "std")] #[serde(rename_all = "camelCase")] @@ -138,6 +139,7 @@ fn impl_build_storage( quote!{ #[cfg(feature = "std")] impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { + /// Build the storage for this module. pub fn build_storage #fn_generic (&self) -> std::result::Result< #scrate::sp_runtime::Storage, String diff --git a/frame/support/procedural/tools/Cargo.toml b/frame/support/procedural/tools/Cargo.toml index ab24e371d3f0efde9f5a48e7de0615504dc347b3..f199f1245db0499aa90425ac6da3966fa2132201 100644 --- a/frame/support/procedural/tools/Cargo.toml +++ b/frame/support/procedural/tools/Cargo.toml @@ -14,3 +14,6 @@ proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full", "visit"] } proc-macro-crate = "0.1.4" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/tools/derive/Cargo.toml b/frame/support/procedural/tools/derive/Cargo.toml index 39ac99e4fdb5950591f90c5534b916deb4c34236..bf6346ab1b67a63c57fb021550dec9a8fda0420a 100644 --- a/frame/support/procedural/tools/derive/Cargo.toml +++ b/frame/support/procedural/tools/derive/Cargo.toml @@ -15,3 +15,6 @@ proc-macro = true proc-macro2 = "1.0.6" quote = { version = "1.0.3", features = ["proc-macro"] } syn = { version = "1.0.7", features = ["proc-macro" ,"full", "extra-traits", "parsing"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index a9c48097ad64ab6ecb28a10c7f7c66dd7b30cf18..aadcec67a3606e0d801efb60965cc27d87624201 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -25,18 +25,35 @@ pub use frame_metadata::{ }; pub use crate::weights::{ SimpleDispatchInfo, GetDispatchInfo, DispatchInfo, WeighData, ClassifyDispatch, - TransactionPriority, Weight, WeighBlock, PaysFee, + TransactionPriority, Weight, PaysFee, PostDispatchInfo, WithPostDispatchInfo, }; -pub use sp_runtime::{traits::Dispatchable, DispatchError, DispatchResult}; +pub use sp_runtime::{traits::Dispatchable, DispatchError}; pub use crate::traits::{CallMetadata, GetCallMetadata, GetCallName}; +/// The return typ of a `Dispatchable` in frame. When returned explicitly from +/// a dispatchable function it allows overriding the default `PostDispatchInfo` +/// returned from a dispatch. +pub type DispatchResultWithPostInfo = + sp_runtime::DispatchResultWithInfo; + +/// Unaugmented version of `DispatchResultWithPostInfo` that can be returned from +/// dispatchable functions and is automatically converted to the augmented type. Should be +/// used whenever the `PostDispatchInfo` does not need to be overwritten. As this should +/// be the common case it is the implicit return type when none is specified. +pub type DispatchResult = Result<(), sp_runtime::DispatchError>; + +/// The error type contained in a `DispatchResultWithPostInfo`. +pub type DispatchErrorWithPostInfo = + sp_runtime::DispatchErrorWithPostInfo; + + /// A type that cannot be instantiated. pub enum Never {} /// Serializable version of Dispatchable. /// This value can be used as a "function" in an extrinsic. pub trait Callable { - type Call: Dispatchable + Codec + Clone + PartialEq + Eq; + type Call: Dispatchable + Codec + Clone + PartialEq + Eq; } // dirty hack to work around serde_derive issue @@ -57,12 +74,14 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; +/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { /// /// // Private functions are dispatchable, but not available to other /// // FRAME pallets. +/// #[weight = SimpleDispatchInfo::default()] /// fn my_function(origin, var: u64) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) @@ -70,6 +89,7 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// /// // Public functions are both dispatchable and available to other /// // FRAME pallets. +/// #[weight = SimpleDispatchInfo::default()] /// pub fn my_public_function(origin) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) @@ -97,15 +117,17 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; +/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { -/// +/// #[weight = SimpleDispatchInfo::default()] /// fn my_long_function(origin) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) /// } /// +/// #[weight = SimpleDispatchInfo::default()] /// fn my_short_function(origin) { /// // Your implementation /// } @@ -114,6 +136,44 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # fn main() {} /// ``` /// +/// ### Consuming only portions of the annotated static weight +/// +/// Per default a callable function consumes all of its static weight as declared via +/// the #[weight] attribute. However, there are use cases where only a portion of this +/// weight should be consumed. In that case the static weight is charged pre dispatch and +/// the difference is refunded post dispatch. +/// +/// In order to make use of this feature the function must return `DispatchResultWithPostInfo` +/// in place of the default `DispatchResult`. Then the actually consumed weight can be returned. +/// To consume a non default weight while returning an error +/// [`WithPostDispatchInfo::with_weight`](./weight/trait.WithPostDispatchInfo.html) can be used +/// to augment any error with custom weight information. +/// +/// ``` +/// # #[macro_use] +/// # extern crate frame_support; +/// # use frame_support::dispatch::{DispatchResultWithPostInfo, WithPostDispatchInfo}; +/// # use frame_support::weights::SimpleDispatchInfo; +/// # use frame_system::{self as system, Trait, ensure_signed}; +/// decl_module! { +/// pub struct Module for enum Call where origin: T::Origin { +/// #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] +/// fn my_long_function(origin, do_expensive_calc: bool) -> DispatchResultWithPostInfo { +/// ensure_signed(origin).map_err(|e| e.with_weight(100_000))?; +/// if do_expensive_calc { +/// // do the expensive calculation +/// // ... +/// // return None to indicate that we are using all weight (the default) +/// return Ok(None.into()); +/// } +/// // expensive calculation not executed: use only a portion of the weight +/// Ok(Some(100_000).into()) +/// } +/// } +/// } +/// # fn main() {} +/// ``` +/// /// ### Privileged Function Example /// /// A privileged function checks that the origin of the call is `ROOT`. @@ -122,9 +182,11 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; +/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed, ensure_root}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { +/// #[weight = SimpleDispatchInfo::default()] /// fn my_privileged_function(origin) -> dispatch::DispatchResult { /// ensure_root(origin)?; /// // Your implementation @@ -200,13 +262,23 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// is a runtime upgrade. This allows each module to upgrade its storage before the storage items are used. /// As such, **calling other modules must be avoided**!! Using this function will implement the /// [`OnRuntimeUpgrade`](../sp_runtime/traits/trait.OnRuntimeUpgrade.html) trait. +/// Function signature must be `fn on_runtime_upgrade() -> frame_support::weights::Weight`. +/// /// * `on_initialize`: Executes at the beginning of a block. Using this function will -/// implement the [`OnInitialize`](../sp_runtime/traits/trait.OnInitialize.html) trait. +/// implement the [`OnInitialize`](./trait.OnInitialize.html) trait. +/// Function signature can be either: +/// * `fn on_initialize(n: BlockNumber) -> frame_support::weights::Weight` or +/// * `fn on_initialize() -> frame_support::weights::Weight` +/// /// * `on_finalize`: Executes at the end of a block. Using this function will -/// implement the [`OnFinalize`](../sp_runtime/traits/trait.OnFinalize.html) trait. +/// implement the [`OnFinalize`](./traits/trait.OnFinalize.html) trait. +/// Function signature can be either: +/// * `fn on_finalize(n: BlockNumber) -> frame_support::weights::Weight` or +/// * `fn on_finalize() -> frame_support::weights::Weight` +/// /// * `offchain_worker`: Executes at the beginning of a block and produces extrinsics for a future block /// upon completion. Using this function will implement the -/// [`OffchainWorker`](../sp_runtime/traits/trait.OffchainWorker.html) trait. +/// [`OffchainWorker`](./traits/trait.OffchainWorker.html) trait. #[macro_export] macro_rules! decl_module { // Entry point #1. @@ -327,7 +399,7 @@ macro_rules! decl_module { "`deposit_event` function is reserved and must follow the syntax: `$vis:vis fn deposit_event() = default;`" ); }; - // Add on_finalize, without a given weight. + // Add on_finalize (@normalize $(#[$attr:meta])* pub struct $mod_type:ident<$trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)?> @@ -354,7 +426,6 @@ macro_rules! decl_module { { $( $on_initialize )* } { $( $on_runtime_upgrade )* } { - #[weight = $crate::dispatch::SimpleDispatchInfo::zero()] fn on_finalize( $( $param_name : $param ),* ) { $( $impl )* } } { $( $offchain )* } @@ -364,7 +435,7 @@ macro_rules! decl_module { $($rest)* ); }; - // Add on_finalize, given weight. + // compile_error on_finalize, given weight removed syntax. (@normalize $(#[$attr:meta])* pub struct $mod_type:ident<$trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)?> @@ -383,26 +454,12 @@ macro_rules! decl_module { fn on_finalize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } $($rest:tt)* ) => { - $crate::decl_module!(@normalize - $(#[$attr])* - pub struct $mod_type<$trait_instance: $trait_name$(, I: $instantiable $(= $module_default_instance)?)?> - for enum $call_type where origin: $origin_type, system = $system - { $( $other_where_bounds )* } - { $( $deposit_event )* } - { $( $on_initialize )* } - { $( $on_runtime_upgrade )* } - { - #[weight = $weight] - fn on_finalize( $( $param_name : $param ),* ) { $( $impl )* } - } - { $( $offchain )* } - { $( $constants )* } - { $( $error_type )* } - [ $( $dispatchables )* ] - $($rest)* + compile_error!( + "`on_finalize` can't be given weight attribute anymore, weight must be returned by \ + `on_initialize` or `on_runtime_upgrade` instead" ); }; - // Add on_runtime_upgrade, without a given weight. + // compile_error on_runtime_upgrade, without a given weight removed syntax. (@normalize $(#[$attr:meta])* pub struct $mod_type:ident< @@ -422,26 +479,11 @@ macro_rules! decl_module { fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } $($rest:tt)* ) => { - $crate::decl_module!(@normalize - $(#[$attr])* - pub struct $mod_type<$trait_instance: $trait_name$(, I: $instantiable $(= $module_default_instance)?)?> - for enum $call_type where origin: $origin_type, system = $system - { $( $other_where_bounds )* } - { $( $deposit_event )* } - { $( $on_initialize )* } - { - #[weight = $crate::dispatch::SimpleDispatchInfo::zero()] - fn on_runtime_upgrade( $( $param_name : $param ),* ) { $( $impl )* } - } - { $( $on_finalize )* } - { $( $offchain )* } - { $( $constants )* } - { $( $error_type )* } - [ $( $dispatchables )* ] - $($rest)* + compile_error!( + "`on_runtime_upgrade` must return Weight, signature has changed." ); }; - // Add on_runtime_upgrade, given weight. + // compile_error on_runtime_upgrade, given weight removed syntax. (@normalize $(#[$attr:meta])* pub struct $mod_type:ident< @@ -461,6 +503,31 @@ macro_rules! decl_module { #[weight = $weight:expr] fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } $($rest:tt)* + ) => { + compile_error!( + "`on_runtime_upgrade` can't be given weight attribute anymore, weight must be returned \ + by the function directly." + ); + }; + // Add on_runtime_upgrade + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + {} + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } + $($rest:tt)* ) => { $crate::decl_module!(@normalize $(#[$attr])* @@ -470,8 +537,7 @@ macro_rules! decl_module { { $( $deposit_event )* } { $( $on_initialize )* } { - #[weight = $weight] - fn on_runtime_upgrade( $( $param_name : $param ),* ) { $( $impl )* } + fn on_runtime_upgrade( $( $param_name : $param ),* ) -> $return { $( $impl )* } } { $( $on_finalize )* } { $( $offchain )* } @@ -481,7 +547,7 @@ macro_rules! decl_module { $($rest)* ); }; - // Add on_initialize, without a given weight. + // compile_error on_initialize, without a given weight removed syntax. (@normalize $(#[$attr:meta])* pub struct $mod_type:ident< @@ -501,26 +567,11 @@ macro_rules! decl_module { fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } $($rest:tt)* ) => { - $crate::decl_module!(@normalize - $(#[$attr])* - pub struct $mod_type<$trait_instance: $trait_name$(, I: $instantiable $(= $module_default_instance)?)?> - for enum $call_type where origin: $origin_type, system = $system - { $( $other_where_bounds )* } - { $( $deposit_event )* } - { - #[weight = $crate::dispatch::SimpleDispatchInfo::zero()] - fn on_initialize( $( $param_name : $param ),* ) { $( $impl )* } - } - { $( $on_runtime_upgrade )* } - { $( $on_finalize )* } - { $( $offchain )* } - { $( $constants )* } - { $( $error_type )* } - [ $( $dispatchables )* ] - $($rest)* + compile_error!( + "`on_initialize` must return Weight, signature has changed." ); }; - // Add on_initialize, given weight. + // compile_error on_initialize, with given weight removed syntax. (@normalize $(#[$attr:meta])* pub struct $mod_type:ident< @@ -540,6 +591,31 @@ macro_rules! decl_module { #[weight = $weight:expr] fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } $($rest:tt)* + ) => { + compile_error!( + "`on_initialize` can't be given weight attribute anymore, weight must be returned \ + by the function directly." + ); + }; + // Add on_initialize + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + {} + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } + $($rest:tt)* ) => { $crate::decl_module!(@normalize $(#[$attr])* @@ -548,8 +624,7 @@ macro_rules! decl_module { { $( $other_where_bounds )* } { $( $deposit_event )* } { - #[weight = $weight] - fn on_initialize( $( $param_name : $param ),* ) { $( $impl )* } + fn on_initialize( $( $param_name : $param ),* ) -> $return { $( $impl )* } } { $( $on_runtime_upgrade )* } { $( $on_finalize )* } @@ -794,27 +869,10 @@ macro_rules! decl_module { ) $( -> $result:ty )* { $( $impl:tt )* } $($rest:tt)* ) => { - $crate::decl_module!(@normalize - $(#[$attr])* - pub struct $mod_type< - $trait_instance: $trait_name$(, $instance: $instantiable $(= $module_default_instance)?)? - > - for enum $call_type where origin: $origin_type, system = $system - { $( $other_where_bounds )* } - { $( $deposit_event )* } - { $( $on_initialize )* } - { $( $on_runtime_upgrade )* } - { $( $on_finalize )* } - { $( $offchain )* } - { $( $constants )* } - { $( $error_type )* } - [ $( $dispatchables )* ] - $(#[doc = $doc_attr])* - #[weight = $crate::dispatch::SimpleDispatchInfo::default()] - $fn_vis fn $fn_name( - $from $(, $(#[$codec_attr])* $param_name : $param )* - ) $( -> $result )* { $( $impl )* } - $($rest)* + compile_error!(concat!( + "Missing weight for ", stringify!($ident), + ". Every dispatchable must have a #[weight] attribute." + ) ); }; // Ignore any ident which is not `origin` with type `T::Origin`. @@ -935,7 +993,7 @@ macro_rules! decl_module { $ignore:ident $mod_type:ident<$trait_instance:ident $(, $instance:ident)?> $fn_name:ident $origin:ident $system:ident [ $( $param_name:ident),* ] ) => { - <$mod_type<$trait_instance $(, $instance)?>>::$fn_name( $origin $(, $param_name )* ) + <$mod_type<$trait_instance $(, $instance)?>>::$fn_name( $origin $(, $param_name )* ).map(Into::into).map_err(Into::into) }; // no `deposit_event` function wanted @@ -965,14 +1023,13 @@ macro_rules! decl_module { (@impl_on_initialize $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } - #[weight = $weight:expr] - fn on_initialize() { $( $impl:tt )* } + fn on_initialize() -> $return:ty { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnInitialize<$trait_instance::BlockNumber> + $crate::traits::OnInitialize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { - fn on_initialize(_block_number_not_used: $trait_instance::BlockNumber) { + fn on_initialize(_block_number_not_used: $trait_instance::BlockNumber) -> $return { use $crate::sp_std::if_std; if_std! { use $crate::tracing; @@ -987,14 +1044,13 @@ macro_rules! decl_module { (@impl_on_initialize $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } - #[weight = $weight:expr] - fn on_initialize($param:ident : $param_ty:ty) { $( $impl:tt )* } + fn on_initialize($param:ident : $param_ty:ty) -> $return:ty { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnInitialize<$trait_instance::BlockNumber> + $crate::traits::OnInitialize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { - fn on_initialize($param: $param_ty) { + fn on_initialize($param: $param_ty) -> $return { use $crate::sp_std::if_std; if_std! { use $crate::tracing; @@ -1011,7 +1067,7 @@ macro_rules! decl_module { { $( $other_where_bounds:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnInitialize<$trait_instance::BlockNumber> + $crate::traits::OnInitialize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* {} }; @@ -1019,14 +1075,13 @@ macro_rules! decl_module { (@impl_on_runtime_upgrade $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } - #[weight = $weight:expr] - fn on_runtime_upgrade() { $( $impl:tt )* } + fn on_runtime_upgrade() -> $return:ty { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnRuntimeUpgrade + $crate::traits::OnRuntimeUpgrade for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { - fn on_runtime_upgrade() { + fn on_runtime_upgrade() -> $return { use $crate::sp_std::if_std; if_std! { use $crate::tracing; @@ -1043,7 +1098,7 @@ macro_rules! decl_module { { $( $other_where_bounds:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnRuntimeUpgrade + $crate::traits::OnRuntimeUpgrade for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* {} }; @@ -1052,11 +1107,10 @@ macro_rules! decl_module { (@impl_on_finalize $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } - #[weight = $weight:expr] fn on_finalize() { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnFinalize<$trait_instance::BlockNumber> + $crate::traits::OnFinalize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_finalize(_block_number_not_used: $trait_instance::BlockNumber) { @@ -1074,11 +1128,10 @@ macro_rules! decl_module { (@impl_on_finalize $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } - #[weight = $weight:expr] fn on_finalize($param:ident : $param_ty:ty) { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnFinalize<$trait_instance::BlockNumber> + $crate::traits::OnFinalize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_finalize($param: $param_ty) { @@ -1098,57 +1151,19 @@ macro_rules! decl_module { { $( $other_where_bounds:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OnFinalize<$trait_instance::BlockNumber> + $crate::traits::OnFinalize<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { } }; - (@impl_block_hooks_weight - $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; - { $( $other_where_bounds:tt )* } - @runtime_upgrade $( - #[weight = $weight_runtime_update:expr] - fn on_runtime_upgrade($( $param_runtime_upgrade:ident : $param_ty_runtime_upgrade:ty )*) { $( $impl_runtime_upgrade:tt )* } - )? - @init $( - #[weight = $weight_initialize:expr] - fn on_initialize($( $param_initialize:ident : $param_ty_initialize:ty )*) { $( $impl_initialize:tt )* } - )? - @fin $( - #[weight = $weight_finalize:expr] - fn on_finalize($( $param_finalize:ident : $param_ty_finalize:ty )*) { $( $impl_finalize:tt )* } - )? - ) => { - impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::dispatch::WeighBlock<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where - $( $other_where_bounds )* - { - $( - fn on_runtime_upgrade() -> $crate::dispatch::Weight { - >::weigh_data(&$weight_initialize, ()) - } - )? - $( - fn on_initialize(n: $trait_instance::BlockNumber) -> $crate::dispatch::Weight { - >::weigh_data(&$weight_initialize, n) - } - )? - $( - fn on_finalize(n: $trait_instance::BlockNumber) -> $crate::dispatch::Weight { - >::weigh_data(&$weight_finalize, n) - } - )? - } - }; - (@impl_offchain $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; { $( $other_where_bounds:tt )* } fn offchain_worker() { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OffchainWorker<$trait_instance::BlockNumber> + $crate::traits::OffchainWorker<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn offchain_worker(_block_number_not_used: $trait_instance::BlockNumber) { $( $impl )* } @@ -1161,7 +1176,7 @@ macro_rules! decl_module { fn offchain_worker($param:ident : $param_ty:ty) { $( $impl:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OffchainWorker<$trait_instance::BlockNumber> + $crate::traits::OffchainWorker<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn offchain_worker($param: $param_ty) { $( $impl )* } @@ -1173,7 +1188,7 @@ macro_rules! decl_module { { $( $other_where_bounds:tt )* } ) => { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> - $crate::sp_runtime::traits::OffchainWorker<$trait_instance::BlockNumber> + $crate::traits::OffchainWorker<$trait_instance::BlockNumber> for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* {} }; @@ -1402,15 +1417,6 @@ macro_rules! decl_module { $( $on_finalize )* } - $crate::decl_module! { - @impl_block_hooks_weight - $mod_type<$trait_instance: $trait_name $(, $instance: $instantiable)?>; - { $( $other_where_bounds )* } - @runtime_upgrade $( $on_runtime_upgrade )* - @init $( $on_initialize )* - @fin $( $on_finalize )* - } - $crate::decl_module! { @impl_offchain $mod_type<$trait_instance: $trait_name $(, $instance: $instantiable)?>; @@ -1483,9 +1489,9 @@ macro_rules! decl_module { &$weight, ($( $param_name, )*) ); - $crate::dispatch::DispatchInfo { - weight, - class, + $crate::dispatch::DispatchInfo { + weight, + class, pays_fee, } }, @@ -1587,7 +1593,9 @@ macro_rules! decl_module { { type Trait = $trait_instance; type Origin = $origin_type; - fn dispatch(self, _origin: Self::Origin) -> $crate::sp_runtime::DispatchResult { + type Info = $crate::weights::DispatchInfo; + type PostInfo = $crate::weights::PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) -> $crate::dispatch::DispatchResultWithPostInfo { match self { $( $call_type::$fn_name( $( $param_name ),* ) => { @@ -1612,10 +1620,10 @@ macro_rules! decl_module { where $( $other_where_bounds )* { #[doc(hidden)] - pub fn dispatch>( + pub fn dispatch>( d: D, origin: D::Origin - ) -> $crate::sp_runtime::DispatchResult { + ) -> $crate::dispatch::DispatchResultWithPostInfo { d.dispatch(origin) } } @@ -1713,10 +1721,12 @@ macro_rules! impl_outer_dispatch { impl $crate::dispatch::Dispatchable for $call_type { type Origin = $origin; type Trait = $call_type; + type Info = $crate::weights::DispatchInfo; + type PostInfo = $crate::weights::PostDispatchInfo; fn dispatch( self, origin: $origin, - ) -> $crate::sp_runtime::DispatchResult { + ) -> $crate::dispatch::DispatchResultWithPostInfo { $crate::impl_outer_dispatch! { @DISPATCH_MATCH self @@ -2076,9 +2086,10 @@ macro_rules! __check_reserved_fn_name { #[allow(dead_code)] mod tests { use super::*; - use crate::sp_runtime::traits::{OnInitialize, OnFinalize, OnRuntimeUpgrade}; use crate::weights::{DispatchInfo, DispatchClass}; - use crate::traits::{CallMetadata, GetCallMetadata, GetCallName}; + use crate::traits::{ + CallMetadata, GetCallMetadata, GetCallName, OnInitialize, OnFinalize, OnRuntimeUpgrade + }; pub trait Trait: system::Trait + Sized where Self::AccountId: From { type Origin; @@ -2098,35 +2109,34 @@ mod tests { } } - struct BlockWeight; - impl> WeighData for BlockWeight { - fn weigh_data(&self, target: BlockNumber) -> Weight { - let target: u32 = target.into(); - if target % 2 == 0 { 10 } else { 0 } - } - } - decl_module! { pub struct Module for enum Call where origin: T::Origin, T::AccountId: From { /// Hi, this is a comment. + #[weight = SimpleDispatchInfo::default()] fn aux_0(_origin) -> DispatchResult { unreachable!() } + + #[weight = SimpleDispatchInfo::default()] fn aux_1(_origin, #[compact] _data: u32,) -> DispatchResult { unreachable!() } + + #[weight = SimpleDispatchInfo::default()] fn aux_2(_origin, _data: i32, _data2: String) -> DispatchResult { unreachable!() } + #[weight = SimpleDispatchInfo::FixedNormal(3)] fn aux_3(_origin) -> DispatchResult { unreachable!() } + + #[weight = SimpleDispatchInfo::default()] fn aux_4(_origin, _data: i32) -> DispatchResult { unreachable!() } - fn aux_5(_origin, _data: i32, #[compact] _data2: u32,) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::FixedNormal(7)] - fn on_initialize(n: T::BlockNumber,) { if n.into() == 42 { panic!("on_initialize") } } - #[weight = BlockWeight] - fn on_finalize(n: T::BlockNumber) { if n.into() == 42 { panic!("on_finalize") } } - #[weight = SimpleDispatchInfo::FixedOperational(69)] - fn on_runtime_upgrade() { } - fn offchain_worker() {} + #[weight = SimpleDispatchInfo::default()] + fn aux_5(_origin, _data: i32, #[compact] _data2: u32,) -> DispatchResult { unreachable!() } #[weight = SimpleDispatchInfo::FixedOperational(5)] - fn operational(_origin,) { unreachable!() } + fn operational(_origin) { unreachable!() } + + fn on_initialize(n: T::BlockNumber,) -> Weight { if n.into() == 42 { panic!("on_initialize") } 7 } + fn on_finalize(n: T::BlockNumber,) { if n.into() == 42 { panic!("on_finalize") } } + fn on_runtime_upgrade() -> Weight { 10 } + fn offchain_worker() {} } } @@ -2254,10 +2264,15 @@ mod tests { #[test] #[should_panic(expected = "on_initialize")] - fn on_initialize_should_work() { + fn on_initialize_should_work_1() { as OnInitialize>::on_initialize(42); } + #[test] + fn on_initialize_should_work_2() { + assert_eq!( as OnInitialize>::on_initialize(10), 7); + } + #[test] #[should_panic(expected = "on_finalize")] fn on_finalize_should_work() { @@ -2266,7 +2281,7 @@ mod tests { #[test] fn on_runtime_upgrade_should_work() { - as OnRuntimeUpgrade>::on_runtime_upgrade(); + assert_eq!( as OnRuntimeUpgrade>::on_runtime_upgrade(), 10); } #[test] @@ -2288,18 +2303,6 @@ mod tests { ); } - #[test] - fn weight_for_block_hooks() { - // independent of block number - assert_eq!(>::on_initialize(0), 7); - assert_eq!(>::on_initialize(10), 7); - assert_eq!(>::on_initialize(100), 7); - - // dependent - assert_eq!(>::on_finalize(2), 10); - assert_eq!(>::on_finalize(3), 0); - } - #[test] fn call_name() { let name = Call::::aux_3().get_call_name(); diff --git a/frame/support/src/error.rs b/frame/support/src/error.rs index 3b105e979d09affe2efe0a2a5b81cf99d474201f..f619250726deda5caea1f76894d1a75793458f01 100644 --- a/frame/support/src/error.rs +++ b/frame/support/src/error.rs @@ -35,6 +35,7 @@ pub use frame_metadata::{ModuleErrorMetadata, ErrorMetadata, DecodeDifferent}; /// /// ``` /// # use frame_support::{decl_error, decl_module}; +/// # use frame_support::weights::SimpleDispatchInfo; /// decl_error! { /// /// Errors that can occur in my module. /// pub enum MyError for Module { @@ -54,6 +55,7 @@ pub use frame_metadata::{ModuleErrorMetadata, ErrorMetadata, DecodeDifferent}; /// pub struct Module for enum Call where origin: T::Origin { /// type Error = MyError; /// +/// #[weight = SimpleDispatchInfo::default()] /// fn do_something(origin) -> frame_support::dispatch::DispatchResult { /// Err(MyError::::YouAreNotCoolEnough.into()) /// } diff --git a/frame/support/src/event.rs b/frame/support/src/event.rs index 8b7de01159df3e1417dfdb82c08f684237442f13..1184b379f44469d72278dd1562223ef51631c6d4 100644 --- a/frame/support/src/event.rs +++ b/frame/support/src/event.rs @@ -288,6 +288,7 @@ macro_rules! __decl_generic_event { } impl<$( $generic_param ),* $(, $instance)?> RawEvent<$( $generic_param ),* $(, $instance)?> { #[allow(dead_code)] + #[doc(hidden)] pub fn metadata() -> &'static [$crate::event::EventMetadata] { $crate::__events_to_metadata!(; $( $events )* ) } diff --git a/frame/support/src/hash.rs b/frame/support/src/hash.rs index f8d6409060f1910700afefb746739c373d8c2b49..693e929a309e366b0605805d1721429904d04806 100644 --- a/frame/support/src/hash.rs +++ b/frame/support/src/hash.rs @@ -92,6 +92,10 @@ impl StorageHasher for Twox64Concat { } impl ReversibleStorageHasher for Twox64Concat { fn reverse(x: &[u8]) -> &[u8] { + if x.len() < 8 { + crate::debug::error!("Invalid reverse: hash length too short"); + return &[] + } &x[8..] } } @@ -110,6 +114,10 @@ impl StorageHasher for Blake2_128Concat { } impl ReversibleStorageHasher for Blake2_128Concat { fn reverse(x: &[u8]) -> &[u8] { + if x.len() < 16 { + crate::debug::error!("Invalid reverse: hash length too short"); + return &[] + } &x[16..] } } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index f242efecc401e098cfbc8b918eba5e7222a782d3..d37a438fc6f8178b86470a92a20f33169ca4b0d5 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -140,6 +140,8 @@ macro_rules! ord_parameter_types { fn contains(t: &$type) -> bool { &$value == t } fn sorted_members() -> $crate::sp_std::prelude::Vec<$type> { vec![$value] } fn count() -> usize { 1 } + #[cfg(feature = "runtime-benchmarks")] + fn add(_: &$type) {} } } } @@ -209,7 +211,11 @@ macro_rules! assert_err { #[cfg(feature = "std")] macro_rules! assert_ok { ( $x:expr $(,)? ) => { - assert_eq!($x, Ok(())); + let is = $x; + match is { + Ok(_) => (), + _ => assert!(false, "Expected Ok(_). Got {:#?}", is), + } }; ( $x:expr, $y:expr $(,)? ) => { assert_eq!($x, Ok($y)); diff --git a/frame/support/src/metadata.rs b/frame/support/src/metadata.rs index 46662e53548a51d8cda0def166d2126a7c0c2216..3d99ddaa84ff4f62fc008a34c7b8185f92b675de 100644 --- a/frame/support/src/metadata.rs +++ b/frame/support/src/metadata.rs @@ -256,9 +256,8 @@ mod tests { struct TestExtension; impl sp_runtime::traits::SignedExtension for TestExtension { type AccountId = u32; - type Call = u32; + type Call = (); type AdditionalSigned = u32; - type DispatchInfo = (); type Pre = (); const IDENTIFIER: &'static str = "testextension"; fn additional_signed(&self) -> Result { @@ -270,9 +269,8 @@ mod tests { struct TestExtension2; impl sp_runtime::traits::SignedExtension for TestExtension2 { type AccountId = u32; - type Call = u32; + type Call = (); type AdditionalSigned = u32; - type DispatchInfo = (); type Pre = (); const IDENTIFIER: &'static str = "testextension2"; fn additional_signed(&self) -> Result { @@ -336,6 +334,7 @@ mod tests { mod event_module { use crate::dispatch::DispatchResult; + use crate::weights::SimpleDispatchInfo; pub trait Trait: super::system::Trait { type Balance; @@ -353,6 +352,7 @@ mod tests { pub struct Module for enum Call where origin: T::Origin { type Error = Error; + #[weight = SimpleDispatchInfo::default()] fn aux_0(_origin) -> DispatchResult { unreachable!() } } } diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 585e17d80d6a0af8ece29e96190c1437a65d075c..7e2040ee234a75fbcd2b488b7ebb40c64bfbefd7 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -19,15 +19,17 @@ //! NOTE: If you're looking for `parameter_types`, it has moved in to the top-level module. use sp_std::{prelude::*, result, marker::PhantomData, ops::Div, fmt::Debug}; -use codec::{FullCodec, Codec, Encode, Decode}; +use codec::{FullCodec, Codec, Encode, Decode, EncodeLike}; use sp_core::u32_trait::Value as U32; use sp_runtime::{ - RuntimeDebug, - ConsensusEngineId, DispatchResult, DispatchError, - traits::{MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput}, + RuntimeDebug, ConsensusEngineId, DispatchResult, DispatchError, traits::{ + MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput, Bounded, Zero, + BadOrigin + }, }; use crate::dispatch::Parameter; use crate::storage::StorageMap; +use impl_trait_for_tuples::impl_for_tuples; /// An abstraction of a value stored within storage, but possibly as part of a larger composite /// item. @@ -86,7 +88,7 @@ impl< Created: Happened, Removed: Happened, K: FullCodec, - T: FullCodec + T: FullCodec, > StoredMap for StorageMapShim { fn get(k: &K) -> T { S::get(k) } fn is_explicit(k: &K) -> bool { S::contains_key(k) } @@ -137,6 +139,35 @@ impl< } } +/// Something that can estimate at which block the next session rotation will happen. This should +/// be the same logical unit that dictates `ShouldEndSession` to the session module. No Assumptions +/// are made about the scheduling of the sessions. +pub trait EstimateNextSessionRotation { + /// Return the block number at which the next session rotation is estimated to happen. + /// + /// None should be returned if the estimation fails to come to an answer + fn estimate_next_session_rotation(now: BlockNumber) -> Option; +} + +impl EstimateNextSessionRotation for () { + fn estimate_next_session_rotation(_: BlockNumber) -> Option { + Default::default() + } +} + +/// Something that can estimate at which block the next `new_session` will be triggered. This must +/// always be implemented by the session module. +pub trait EstimateNextNewSession { + /// Return the block number at which the next new session is estimated to happen. + fn estimate_next_new_session(now: BlockNumber) -> Option; +} + +impl EstimateNextNewSession for () { + fn estimate_next_new_session(_: BlockNumber) -> Option { + Default::default() + } +} + /// Anything that can have a `::len()` method. pub trait Len { /// Return the length of data type. @@ -193,14 +224,14 @@ impl IsDeadAccount for () { } /// Handler for when a new account has been created. -#[impl_trait_for_tuples::impl_for_tuples(30)] +#[impl_for_tuples(30)] pub trait OnNewAccount { /// A new account `who` has been registered. fn on_new_account(who: &AccountId); } /// The account with the given id was reaped. -#[impl_trait_for_tuples::impl_for_tuples(30)] +#[impl_for_tuples(30)] pub trait OnKilledAccount { /// The account with the given id was reaped. fn on_killed_account(who: &AccountId); @@ -857,6 +888,12 @@ pub trait Time { fn now() -> Self::Moment; } +/// Trait to deal with unix time. +pub trait UnixTime { + /// Return duration since `SystemTime::UNIX_EPOCH`. + fn now() -> core::time::Duration; +} + impl WithdrawReasons { /// Choose all variants except for `one`. /// @@ -996,6 +1033,21 @@ impl Randomness for () { } } +/// Trait to be used by block producing consensus engine modules to determine +/// how late the current block is (e.g. in a slot-based proposal mechanism how +/// many slots were skipped since the previous block). +pub trait Lateness { + /// Returns a generic measure of how late the current block is compared to + /// its parent. + fn lateness(&self) -> N; +} + +impl Lateness for () { + fn lateness(&self) -> N { + Zero::zero() + } +} + /// Implementors of this trait provide information about whether or not some validator has /// been registered with them. The [Session module](../../pallet_session/index.html) is an implementor. pub trait ValidatorRegistration { @@ -1042,3 +1094,187 @@ pub trait GetCallMetadata { /// Return a [`CallMetadata`], containing function and pallet name of the Call. fn get_call_metadata(&self) -> CallMetadata; } + +/// The block finalization trait. Implementing this lets you express what should happen +/// for your module when the block is ending. +#[impl_for_tuples(30)] +pub trait OnFinalize { + /// The block is being finalized. Implement to have something happen. + fn on_finalize(_n: BlockNumber) {} +} + +/// The block initialization trait. Implementing this lets you express what should happen +/// for your module when the block is beginning (right before the first extrinsic is executed). +pub trait OnInitialize { + /// The block is being initialized. Implement to have something happen. + /// + /// Return the non-negotiable weight consumed in the block. + fn on_initialize(_n: BlockNumber) -> crate::weights::Weight { 0 } +} + +#[impl_for_tuples(30)] +impl OnInitialize for Tuple { + fn on_initialize(_n: BlockNumber) -> crate::weights::Weight { + let mut weight = 0; + for_tuples!( #( weight = weight.saturating_add(Tuple::on_initialize(_n.clone())); )* ); + weight + } +} + +/// The runtime upgrade trait. Implementing this lets you express what should happen +/// when the runtime upgrades, and changes may need to occur to your module. +pub trait OnRuntimeUpgrade { + /// Perform a module upgrade. + /// + /// Return the non-negotiable weight consumed for runtime upgrade. + fn on_runtime_upgrade() -> crate::weights::Weight { 0 } +} + +#[impl_for_tuples(30)] +impl OnRuntimeUpgrade for Tuple { + fn on_runtime_upgrade() -> crate::weights::Weight { + let mut weight = 0; + for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); + weight + } +} + +/// Off-chain computation trait. +/// +/// Implementing this trait on a module allows you to perform long-running tasks +/// that make (by default) validators generate transactions that feed results +/// of those long-running computations back on chain. +/// +/// NOTE: This function runs off-chain, so it can access the block state, +/// but cannot preform any alterations. More specifically alterations are +/// not forbidden, but they are not persisted in any way after the worker +/// has finished. +#[impl_for_tuples(30)] +pub trait OffchainWorker { + /// This function is being called after every block import (when fully synced). + /// + /// Implement this and use any of the `Offchain` `sp_io` set of APIs + /// to perform off-chain computations, calls and submit transactions + /// with results to trigger any on-chain changes. + /// Any state alterations are lost and are not persisted. + fn offchain_worker(_n: BlockNumber) {} +} + +pub mod schedule { + use super::*; + + /// Information relating to the period of a scheduled task. First item is the length of the + /// period and the second is the number of times it should be executed in total before the task + /// is considered finished and removed. + pub type Period = (BlockNumber, u32); + + /// Priority with which a call is scheduled. It's just a linear amount with lowest values meaning + /// higher priority. + pub type Priority = u8; + + /// The highest priority. We invert the value so that normal sorting will place the highest + /// priority at the beginning of the list. + pub const HIGHEST_PRORITY: Priority = 0; + /// Anything of this value or lower will definitely be scheduled on the block that they ask for, even + /// if it breaches the `MaximumWeight` limitation. + pub const HARD_DEADLINE: Priority = 63; + /// The lowest priority. Most stuff should be around here. + pub const LOWEST_PRORITY: Priority = 255; + + /// A type that can be used as a scheduler. + pub trait Anon { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + Debug; + + /// Schedule a one-off dispatch to happen at the beginning of some block in the future. + /// + /// This is not named. + /// + /// Infallible. + fn schedule( + when: BlockNumber, + maybe_periodic: Option>, + priority: Priority, + call: Call + ) -> Self::Address; + + /// Cancel a scheduled task. If periodic, then it will cancel all further instances of that, + /// also. + /// + /// Will return an error if the `address` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + /// + /// NOTE2: This will not work to cancel periodic tasks after their initial execution. For + /// that, you must name the task explicitly using the `Named` trait. + fn cancel(address: Self::Address) -> Result<(), ()>; + } + + /// A type that can be used as a scheduler. + pub trait Named { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + sp_std::fmt::Debug; + + /// Schedule a one-off dispatch to happen at the beginning of some block in the future. + /// + /// - `id`: The identity of the task. This must be unique and will return an error if not. + fn schedule_named( + id: impl Encode, + when: BlockNumber, + maybe_periodic: Option>, + priority: Priority, + call: Call + ) -> Result; + + /// Cancel a scheduled, named task. If periodic, then it will cancel all further instances + /// of that, also. + /// + /// Will return an error if the `id` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + fn cancel_named(id: impl Encode) -> Result<(), ()>; + } +} + +/// Some sort of check on the origin is performed by this object. +pub trait EnsureOrigin { + /// A return type. + type Success; + /// Perform the origin check. + fn ensure_origin(o: OuterOrigin) -> result::Result { + Self::try_origin(o).map_err(|_| BadOrigin) + } + /// Perform the origin check. + fn try_origin(o: OuterOrigin) -> result::Result; + + /// Returns an outer origin capable of passing `try_origin` check. + /// + /// ** Should be used for benchmarking only!!! ** + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> OuterOrigin; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn on_initialize_and_on_runtime_upgrade_weight_merge_works() { + struct Test; + impl OnInitialize for Test { + fn on_initialize(_n: u8) -> crate::weights::Weight { + 10 + } + } + impl OnRuntimeUpgrade for Test { + fn on_runtime_upgrade() -> crate::weights::Weight { + 20 + } + } + + assert_eq!(<(Test, Test)>::on_initialize(0), 20); + assert_eq!(<(Test, Test)>::on_runtime_upgrade(), 40); + } +} diff --git a/frame/support/src/unsigned.rs b/frame/support/src/unsigned.rs index 4289e4e474f53971f29f75208d9cf865b6e173a2..3bc6f692affc20426f196792666fe1c0413c146f 100644 --- a/frame/support/src/unsigned.rs +++ b/frame/support/src/unsigned.rs @@ -18,7 +18,7 @@ pub use crate::sp_runtime::traits::ValidateUnsigned; #[doc(hidden)] pub use crate::sp_runtime::transaction_validity::{ - TransactionValidity, UnknownTransaction, TransactionValidityError, + TransactionValidity, UnknownTransaction, TransactionValidityError, TransactionSource, }; @@ -34,7 +34,8 @@ pub use crate::sp_runtime::transaction_validity::{ /// # impl frame_support::unsigned::ValidateUnsigned for Module { /// # type Call = Call; /// # -/// # fn validate_unsigned(call: &Self::Call) -> frame_support::unsigned::TransactionValidity { +/// # fn validate_unsigned(_source: frame_support::unsigned::TransactionSource, _call: &Self::Call) +/// -> frame_support::unsigned::TransactionValidity { /// # unimplemented!(); /// # } /// # } @@ -78,10 +79,14 @@ macro_rules! impl_outer_validate_unsigned { } } - fn validate_unsigned(call: &Self::Call) -> $crate::unsigned::TransactionValidity { + fn validate_unsigned( + #[allow(unused_variables)] + source: $crate::unsigned::TransactionSource, + call: &Self::Call, + ) -> $crate::unsigned::TransactionValidity { #[allow(unreachable_patterns)] match call { - $( Call::$module(inner_call) => $module::validate_unsigned(inner_call), )* + $( Call::$module(inner_call) => $module::validate_unsigned(source, inner_call), )* _ => $crate::unsigned::UnknownTransaction::NoUnsignedValidator.into(), } } @@ -110,7 +115,10 @@ mod test_partial_and_full_call { impl super::super::ValidateUnsigned for Module { type Call = Call; - fn validate_unsigned(_call: &Self::Call) -> super::super::TransactionValidity { + fn validate_unsigned( + _source: super::super::TransactionSource, + _call: &Self::Call + ) -> super::super::TransactionValidity { unimplemented!(); } } diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index 8926ed949302b304935a5df7431eae09a886eb57..ea3368550f301b96f3dd82bc15daf2d000a5b60f 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -37,14 +37,14 @@ #[cfg(feature = "std")] use serde::{Serialize, Deserialize}; -use impl_trait_for_tuples::impl_for_tuples; use codec::{Encode, Decode}; -use sp_arithmetic::traits::{Bounded, Zero}; +use sp_arithmetic::traits::Bounded; use sp_runtime::{ RuntimeDebug, traits::SignedExtension, generic::{CheckedExtrinsic, UncheckedExtrinsic}, }; +use crate::dispatch::{DispatchErrorWithPostInfo, DispatchError}; /// Re-export priority as type pub use sp_runtime::transaction_validity::TransactionPriority; @@ -67,17 +67,6 @@ pub trait ClassifyDispatch { fn classify_dispatch(&self, target: T) -> DispatchClass; } -/// Means of determining the weight of a block's life cycle hooks: `on_initialize`, `on_finalize`, -/// `on_runtime_upgrade`, and such. -pub trait WeighBlock { - /// Return the weight of the block's on_runtime_upgrade hook. - fn on_runtime_upgrade() -> Weight { Zero::zero() } - /// Return the weight of the block's on_initialize hook. - fn on_initialize(_: BlockNumber) -> Weight { Zero::zero() } - /// Return the weight of the block's on_finalize hook. - fn on_finalize(_: BlockNumber) -> Weight { Zero::zero() } -} - /// Indicates if dispatch function should pay fees or not. /// If set to false, the block resource limits are applied, yet no fee is deducted. pub trait PaysFee { @@ -86,34 +75,6 @@ pub trait PaysFee { } } -/// Maybe I can do something to remove the duplicate code here. -#[impl_for_tuples(30)] -impl WeighBlock for SingleModule { - fn on_runtime_upgrade() -> Weight { - let mut accumulated_weight: Weight = Zero::zero(); - for_tuples!( - #( accumulated_weight = accumulated_weight.saturating_add(SingleModule::on_runtime_upgrade()); )* - ); - accumulated_weight - } - - fn on_initialize(n: BlockNumber) -> Weight { - let mut accumulated_weight: Weight = Zero::zero(); - for_tuples!( - #( accumulated_weight = accumulated_weight.saturating_add(SingleModule::on_initialize(n)); )* - ); - accumulated_weight - } - - fn on_finalize(n: BlockNumber) -> Weight { - let mut accumulated_weight: Weight = Zero::zero(); - for_tuples!( - #( accumulated_weight = accumulated_weight.saturating_add(SingleModule::on_finalize(n)); )* - ); - accumulated_weight - } -} - /// A generalized group of dispatch types. This is only distinguishing normal, user-triggered transactions /// (`Normal`) and anything beyond which serves a higher purpose to the system (`Operational`). #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] @@ -124,6 +85,19 @@ pub enum DispatchClass { Normal, /// An operational dispatch. Operational, + /// A mandatory dispatch. These kinds of dispatch are always included regardless of their + /// weight, therefore it is critical that they are separately validated to ensure that a + /// malicious validator cannot craft a valid but impossibly heavy block. Usually this just means + /// ensuring that the extrinsic can only be included once and that it is always very light. + /// + /// Do *NOT* use it for extrinsics that can be heavy. + /// + /// The only real use case for this is inherent extrinsics that are required to execute in a + /// block for the block to be valid, and it solves the issue in the case that the block + /// initialization is sufficiently heavy to mean that those inherents do not fit into the + /// block. Essentially, we assume that in these exceptional circumstances, it is better to + /// allow an overweight block to be created than to not allow any block at all to be created. + Mandatory, } impl Default for DispatchClass { @@ -141,6 +115,8 @@ impl From for DispatchClass { SimpleDispatchInfo::FixedNormal(_) => DispatchClass::Normal, SimpleDispatchInfo::MaxNormal => DispatchClass::Normal, SimpleDispatchInfo::InsecureFreeNormal => DispatchClass::Normal, + + SimpleDispatchInfo::FixedMandatory(_) => DispatchClass::Mandatory, } } } @@ -156,6 +132,64 @@ pub struct DispatchInfo { pub pays_fee: bool, } +/// Weight information that is only available post dispatch. +#[derive(Clone, Copy, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)] +pub struct PostDispatchInfo { + /// Actual weight consumed by a call or `None` which stands for the worst case static weight. + pub actual_weight: Option, +} + +impl From> for PostDispatchInfo { + fn from(actual_weight: Option) -> Self { + Self { + actual_weight, + } + } +} + +impl From<()> for PostDispatchInfo { + fn from(_: ()) -> Self { + Self { + actual_weight: None, + } + } +} + +impl sp_runtime::traits::Printable for PostDispatchInfo { + fn print(&self) { + "actual_weight=".print(); + match self.actual_weight { + Some(weight) => weight.print(), + None => "max-weight".print(), + } + } +} + +/// Allows easy conversion from `DispatchError` to `DispatchErrorWithPostInfo` for dispatchables +/// that want to return a custom a posteriori weight on error. +pub trait WithPostDispatchInfo { + /// Call this on your modules custom errors type in order to return a custom weight on error. + /// + /// # Example + /// + /// ```ignore + /// let who = ensure_signed(origin).map_err(|e| e.with_weight(100))?; + /// ensure!(who == me, Error::::NotMe.with_weight(200_000)); + /// ``` + fn with_weight(self, actual_weight: Weight) -> DispatchErrorWithPostInfo; +} + +impl WithPostDispatchInfo for T where + T: Into +{ + fn with_weight(self, actual_weight: Weight) -> DispatchErrorWithPostInfo { + DispatchErrorWithPostInfo { + post_info: PostDispatchInfo { actual_weight: Some(actual_weight) }, + error: self.into(), + } + } +} + /// A `Dispatchable` function (aka transaction) that can carry some static information along with /// it, using the `#[weight]` attribute. pub trait GetDispatchInfo { @@ -193,6 +227,11 @@ pub enum SimpleDispatchInfo { FixedOperational(Weight), /// An operational dispatch with the maximum weight. MaxOperational, + /// A mandatory dispatch with fixed weight. + /// + /// NOTE: Signed transactions may not (directly) dispatch this kind of a call, so the other + /// attributes concerning transactability (e.g. priority, fee paying) are moot. + FixedMandatory(Weight), } impl WeighData for SimpleDispatchInfo { @@ -201,9 +240,9 @@ impl WeighData for SimpleDispatchInfo { SimpleDispatchInfo::FixedNormal(w) => *w, SimpleDispatchInfo::MaxNormal => Bounded::max_value(), SimpleDispatchInfo::InsecureFreeNormal => Bounded::min_value(), - SimpleDispatchInfo::FixedOperational(w) => *w, SimpleDispatchInfo::MaxOperational => Bounded::max_value(), + SimpleDispatchInfo::FixedMandatory(w) => *w, } } } @@ -220,9 +259,9 @@ impl PaysFee for SimpleDispatchInfo { SimpleDispatchInfo::FixedNormal(_) => true, SimpleDispatchInfo::MaxNormal => true, SimpleDispatchInfo::InsecureFreeNormal => true, - SimpleDispatchInfo::FixedOperational(_) => true, SimpleDispatchInfo::MaxOperational => true, + SimpleDispatchInfo::FixedMandatory(_) => true, } } } diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 0f7c89f5c026725358435e25c2ebcaa7ae8742b9..773523579b0b4ce8d580e5e80aad796591947fdf 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-io ={ path = "../../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} sp-state-machine = { version = "0.8.0-alpha.5", optional = true, path = "../../../primitives/state-machine" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../" } @@ -32,3 +32,6 @@ std = [ "sp-runtime/std", "sp-state-machine", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/test/tests/decl_error.rs b/frame/support/test/tests/decl_error.rs index 39d7dbdb9647857ce1d054d53acac90fda42d085..4191e79f2417ca59d118e0265094fa4e9a5b8782 100644 --- a/frame/support/test/tests/decl_error.rs +++ b/frame/support/test/tests/decl_error.rs @@ -32,6 +32,7 @@ mod module1 { pub struct Module, I: Instance = DefaultInstance> for enum Call where origin: ::Origin { + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { Err(Error::::Something.into()) } @@ -58,6 +59,7 @@ mod module2 { pub struct Module for enum Call where origin: ::Origin { + #[weight = frame_support::weights::SimpleDispatchInfo::default()] pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { Err(Error::::Something.into()) } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 92281ef1311a79aa4908e13f024042c2a2e07bc8..ea5d32fea3b222b754331aff047d65782b671911 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -55,6 +55,7 @@ mod module1 { fn deposit_event() = default; + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn one(origin) { system::ensure_root(origin)?; Self::deposit_event(RawEvent::AnotherVariant(3)); diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.rs b/frame/support/test/tests/reserved_keyword/on_initialize.rs index 84feb2d93f36cfc2212f44a7fa646d42501355b8..8eacc836c48685c1516ce129780c71f0ee636ea1 100644 --- a/frame/support/test/tests/reserved_keyword/on_initialize.rs +++ b/frame/support/test/tests/reserved_keyword/on_initialize.rs @@ -19,6 +19,7 @@ macro_rules! reserved { frame_support::decl_module! { pub struct Module for enum Call where origin: T::Origin { + #[weight = frame_support::weights::SimpleDispatchInfo::default()] fn $reserved(_origin) -> dispatch::DispatchResult { unreachable!() } } } diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.stderr b/frame/support/test/tests/reserved_keyword/on_initialize.stderr index d20a6e11451e7d52f4ded9a810ddc81f0d58bbb8..e899ef5d789426c620a9c92f7190a2ac5fdabd4f 100644 --- a/frame/support/test/tests/reserved_keyword/on_initialize.stderr +++ b/frame/support/test/tests/reserved_keyword/on_initialize.stderr @@ -1,39 +1,39 @@ error: Invalid call fn name: `on_finalize`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:30:1 + --> $DIR/on_initialize.rs:31:1 | -30 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); +31 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation | = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) error: Invalid call fn name: `on_initialize`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:30:1 + --> $DIR/on_initialize.rs:31:1 | -30 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); +31 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation | = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) error: Invalid call fn name: `on_runtime_upgrade`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:30:1 + --> $DIR/on_initialize.rs:31:1 | -30 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); +31 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation | = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) error: Invalid call fn name: `offchain_worker`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:30:1 + --> $DIR/on_initialize.rs:31:1 | -30 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); +31 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation | = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) -error: `deposit_event` function is reserved and must follow the syntax: `$vis:vis fn deposit_event() = default;` - --> $DIR/on_initialize.rs:30:1 +error: Invalid call fn name: `deposit_event`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. + --> $DIR/on_initialize.rs:31:1 | -30 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); +31 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ in this macro invocation | = note: this error originates in a macro outside of the current crate (in Nightly builds, run with -Z external-macro-backtrace for more info) diff --git a/frame/system/Cargo.toml b/frame/system/Cargo.toml index 92b76555665a73898df8660898c3ed9efa076b2d..78288cff917e233306fe0bee01f5a04d96263f16 100644 --- a/frame/system/Cargo.toml +++ b/frame/system/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME system module" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.5"} @@ -36,8 +36,11 @@ std = [ "sp-runtime/std", "sp-version/std", ] -runtime-benchmarks = [] +runtime-benchmarks = ["sp-runtime/runtime-benchmarks"] [[bench]] name = "bench" harness = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/system/rpc/runtime-api/Cargo.toml b/frame/system/rpc/runtime-api/Cargo.toml index 6df4f0ea610b4b5416065627be8fd903109a0a32..9519427297425504a8fd1b9c769c5c2a6edd899e 100644 --- a/frame/system/rpc/runtime-api/Cargo.toml +++ b/frame/system/rpc/runtime-api/Cargo.toml @@ -10,7 +10,7 @@ description = "Runtime API definition required by System RPC extensions." [dependencies] sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } [features] default = ["std"] @@ -18,3 +18,6 @@ std = [ "sp-api/std", "codec/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 138a15142769896b58a255966242afd5386375dc..a38a8854c75c31f37c6eacfc84b28eb887061565 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -68,13 +68,14 @@ //! ### Example - Get extrinsic count and parent hash for the current block //! //! ``` -//! use frame_support::{decl_module, dispatch}; +//! use frame_support::{decl_module, dispatch, weights::SimpleDispatchInfo}; //! use frame_system::{self as system, ensure_signed}; //! //! pub trait Trait: system::Trait {} //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = SimpleDispatchInfo::default()] //! pub fn system_module_example(origin) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _extrinsic_count = >::extrinsic_count(); @@ -98,7 +99,7 @@ use sp_std::marker::PhantomData; use sp_std::fmt::Debug; use sp_version::RuntimeVersion; use sp_runtime::{ - RuntimeDebug, Perbill, DispatchOutcome, DispatchError, + RuntimeDebug, Perbill, DispatchOutcome, DispatchError, DispatchResult, generic::{self, Era}, transaction_validity::{ ValidTransaction, TransactionPriority, TransactionLongevity, TransactionValidityError, @@ -106,8 +107,9 @@ use sp_runtime::{ }, traits::{ self, CheckEqual, AtLeast32Bit, Zero, SignedExtension, Lookup, LookupError, - SimpleBitOps, Hash, Member, MaybeDisplay, EnsureOrigin, BadOrigin, SaturatedConversion, + SimpleBitOps, Hash, Member, MaybeDisplay, BadOrigin, SaturatedConversion, MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, + Dispatchable, DispatchInfoOf, PostDispatchInfoOf, }, }; @@ -116,9 +118,9 @@ use frame_support::{ decl_module, decl_event, decl_storage, decl_error, storage, Parameter, ensure, debug, traits::{ Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened, - StoredMap, + StoredMap, EnsureOrigin, }, - weights::{Weight, DispatchInfo, DispatchClass, SimpleDispatchInfo, FunctionOf}, + weights::{Weight, DispatchInfo, DispatchClass, SimpleDispatchInfo, FunctionOf} }; use codec::{Encode, Decode, FullCodec, EncodeLike}; @@ -145,7 +147,7 @@ pub trait Trait: 'static + Eq + Clone { + Clone; /// The aggregated `Call` type. - type Call: Debug; + type Call: Dispatchable + Debug; /// Account index (aka nonce) type. This stores the number of previous transactions associated /// with a sender account. @@ -365,7 +367,7 @@ decl_storage! { ExtrinsicData get(fn extrinsic_data): map hasher(twox_64_concat) u32 => Vec; /// The current block number being processed. Set by `execute_block`. - Number get(fn block_number) build(|_| 1.into()): T::BlockNumber; + Number get(fn block_number): T::BlockNumber; /// Hash of the previous block. ParentHash get(fn parent_hash) build(|_| hash69()): T::Hash; @@ -577,12 +579,17 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Root) + } } pub struct EnsureSigned(sp_std::marker::PhantomData); impl< O: Into, O>> + From>, - AccountId, + AccountId: Default, > EnsureOrigin for EnsureSigned { type Success = AccountId; fn try_origin(o: O) -> Result { @@ -591,13 +598,18 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::Signed(Default::default())) + } } pub struct EnsureSignedBy(sp_std::marker::PhantomData<(Who, AccountId)>); impl< O: Into, O>> + From>, Who: Contains, - AccountId: PartialEq + Clone + Ord, + AccountId: PartialEq + Clone + Ord + Default, > EnsureOrigin for EnsureSignedBy { type Success = AccountId; fn try_origin(o: O) -> Result { @@ -606,6 +618,16 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + let members = Who::sorted_members(); + let first_member = match members.get(0) { + Some(account) => account.clone(), + None => Default::default(), + }; + O::from(RawOrigin::Signed(first_member.clone())) + } } pub struct EnsureNone(sp_std::marker::PhantomData); @@ -620,6 +642,11 @@ impl< r => Err(O::from(r)), }) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + O::from(RawOrigin::None) + } } pub struct EnsureNever(sp_std::marker::PhantomData); @@ -628,6 +655,11 @@ impl EnsureOrigin for EnsureNever { fn try_origin(o: O) -> Result { Err(o) } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + unimplemented!() + } } /// Ensure that the origin `o` represents a signed extrinsic (i.e. transaction). @@ -721,6 +753,10 @@ impl Module { /// This will update storage entries that correspond to the specified topics. /// It is expected that light-clients could subscribe to this topics. pub fn deposit_event_indexed(topics: &[T::Hash], event: T::Event) { + let block_number = Self::block_number(); + // Don't populate events on genesis. + if block_number.is_zero() { return } + let phase = ExecutionPhase::get().unwrap_or_default(); let event = EventRecord { phase, @@ -753,10 +789,9 @@ impl Module { return; } - let block_no = Self::block_number(); for topic in topics { // The same applies here. - if >::append(topic, &[(block_no, event_idx)]).is_err() { + if >::append(topic, &[(block_number, event_idx)]).is_err() { return; } } @@ -1133,13 +1168,16 @@ pub fn split_inner(option: Option, splitter: impl FnOnce(T) -> (R, S #[derive(Encode, Decode, Clone, Eq, PartialEq)] pub struct CheckWeight(PhantomData); -impl CheckWeight { +impl CheckWeight where + T::Call: Dispatchable +{ /// Get the quota ratio of each dispatch class type. This indicates that all operational /// dispatches can use the full capacity of any resource, while user-triggered ones can consume /// a portion. fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { match class { - DispatchClass::Operational => ::one(), + DispatchClass::Operational | DispatchClass::Mandatory + => ::one(), DispatchClass::Normal => T::AvailableBlockRatio::get(), } } @@ -1148,14 +1186,14 @@ impl CheckWeight { /// /// Upon successes, it returns the new block weight as a `Result`. fn check_weight( - info: ::DispatchInfo, + info: &DispatchInfoOf, ) -> Result { let current_weight = Module::::all_extrinsics_weight(); let maximum_weight = T::MaximumBlockWeight::get(); let limit = Self::get_dispatch_limit_ratio(info.class) * maximum_weight; let added_weight = info.weight.min(limit); let next_weight = current_weight.saturating_add(added_weight); - if next_weight > limit { + if next_weight > limit && info.class != DispatchClass::Mandatory { Err(InvalidTransaction::ExhaustsResources.into()) } else { Ok(next_weight) @@ -1166,7 +1204,7 @@ impl CheckWeight { /// /// Upon successes, it returns the new block length as a `Result`. fn check_block_length( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { let current_len = Module::::all_extrinsics_len(); @@ -1182,10 +1220,12 @@ impl CheckWeight { } /// get the priority of an extrinsic denoted by `info`. - fn get_priority(info: ::DispatchInfo) -> TransactionPriority { + fn get_priority(info: &DispatchInfoOf) -> TransactionPriority { match info.class { DispatchClass::Normal => info.weight.into(), - DispatchClass::Operational => Bounded::max_value() + DispatchClass::Operational => Bounded::max_value(), + // Mandatory extrinsics are only for inherents; never transactions. + DispatchClass::Mandatory => Bounded::min_value(), } } @@ -1198,7 +1238,7 @@ impl CheckWeight { /// /// It checks and notes the new weight and length. fn do_pre_dispatch( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { let next_len = Self::check_block_length(info, len)?; @@ -1212,7 +1252,7 @@ impl CheckWeight { /// /// It only checks that the block weight and length limit will not exceed. fn do_validate( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { // ignore the next weight and length. If they return `Ok`, then it is below the limit. @@ -1223,11 +1263,12 @@ impl CheckWeight { } } -impl SignedExtension for CheckWeight { +impl SignedExtension for CheckWeight where + T::Call: Dispatchable +{ type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckWeight"; @@ -1237,9 +1278,12 @@ impl SignedExtension for CheckWeight { self, _who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { + if info.class == DispatchClass::Mandatory { + Err(InvalidTransaction::MandatoryDispatch)? + } Self::do_pre_dispatch(info, len) } @@ -1247,15 +1291,18 @@ impl SignedExtension for CheckWeight { &self, _who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { + if info.class == DispatchClass::Mandatory { + Err(InvalidTransaction::MandatoryDispatch)? + } Self::do_validate(info, len) } fn pre_dispatch_unsigned( _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { Self::do_pre_dispatch(info, len) @@ -1263,11 +1310,27 @@ impl SignedExtension for CheckWeight { fn validate_unsigned( _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { Self::do_validate(info, len) } + + fn post_dispatch( + _pre: Self::Pre, + info: &DispatchInfoOf, + _post_info: &PostDispatchInfoOf, + _len: usize, + result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + // Since mandatory dispatched do not get validated for being overweight, we are sensitive + // to them actually being useful. Block producers are thus not allowed to include mandatory + // extrinsics that result in error. + if info.class == DispatchClass::Mandatory && result.is_err() { + Err(InvalidTransaction::BadMandatory)? + } + Ok(()) + } } impl Debug for CheckWeight { @@ -1305,11 +1368,12 @@ impl Debug for CheckNonce { } } -impl SignedExtension for CheckNonce { +impl SignedExtension for CheckNonce where + T::Call: Dispatchable +{ type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckNonce"; @@ -1319,7 +1383,7 @@ impl SignedExtension for CheckNonce { self, who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> Result<(), TransactionValidityError> { let mut account = Account::::get(who); @@ -1341,7 +1405,7 @@ impl SignedExtension for CheckNonce { &self, who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { // check index @@ -1400,7 +1464,6 @@ impl SignedExtension for CheckEra { type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = T::Hash; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckEra"; @@ -1408,7 +1471,7 @@ impl SignedExtension for CheckEra { &self, _who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { let current_u64 = >::block_number().saturated_into::(); @@ -1457,7 +1520,6 @@ impl SignedExtension for CheckGenesis { type AccountId = T::AccountId; type Call = ::Call; type AdditionalSigned = T::Hash; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckGenesis"; @@ -1493,7 +1555,6 @@ impl SignedExtension for CheckVersion { type AccountId = T::AccountId; type Call = ::Call; type AdditionalSigned = u32; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckVersion"; @@ -1557,9 +1618,22 @@ mod tests { fn on_killed_account(who: &u64) { KILLED.with(|r| r.borrow_mut().push(*who)) } } + #[derive(Debug)] + pub struct Call {} + impl Dispatchable for Call { + type Origin = (); + type Trait = (); + type Info = DispatchInfo; + type PostInfo = (); + fn dispatch(self, _origin: Self::Origin) + -> sp_runtime::DispatchResultWithInfo { + panic!("Do not use dummy implementation for dispatch."); + } + } + impl Trait for Test { type Origin = Origin; - type Call = (); + type Call = Call; type Index = u64; type BlockNumber = u64; type Hash = H256; @@ -1592,7 +1666,7 @@ mod tests { type System = Module; - const CALL: &::Call = &(); + const CALL: &::Call = &Call {}; fn new_test_ext() -> sp_io::TestExternalities { GenesisConfig::default().build_storage::().unwrap().into() @@ -1793,14 +1867,14 @@ mod tests { let info = DispatchInfo::default(); let len = 0_usize; // stale - assert!(CheckNonce::(0).validate(&1, CALL, info, len).is_err()); - assert!(CheckNonce::(0).pre_dispatch(&1, CALL, info, len).is_err()); + assert!(CheckNonce::(0).validate(&1, CALL, &info, len).is_err()); + assert!(CheckNonce::(0).pre_dispatch(&1, CALL, &info, len).is_err()); // correct - assert!(CheckNonce::(1).validate(&1, CALL, info, len).is_ok()); - assert!(CheckNonce::(1).pre_dispatch(&1, CALL, info, len).is_ok()); + assert!(CheckNonce::(1).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(1).pre_dispatch(&1, CALL, &info, len).is_ok()); // future - assert!(CheckNonce::(5).validate(&1, CALL, info, len).is_ok()); - assert!(CheckNonce::(5).pre_dispatch(&1, CALL, info, len).is_err()); + assert!(CheckNonce::(5).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(5).pre_dispatch(&1, CALL, &info, len).is_err()); }) } @@ -1825,9 +1899,9 @@ mod tests { if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } }; - reset_check_weight(small, false, 0); - reset_check_weight(medium, false, 0); - reset_check_weight(big, true, 1); + reset_check_weight(&small, false, 0); + reset_check_weight(&medium, false, 0); + reset_check_weight(&big, true, 1); }) } @@ -1838,7 +1912,7 @@ mod tests { let len = 0_usize; assert_eq!(System::all_extrinsics_weight(), 0); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, free, len); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &free, len); assert!(r.is_ok()); assert_eq!(System::all_extrinsics_weight(), 0); }) @@ -1852,7 +1926,7 @@ mod tests { let normal_limit = normal_weight_limit(); assert_eq!(System::all_extrinsics_weight(), 0); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, max, len); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &max, len); assert!(r.is_ok()); assert_eq!(System::all_extrinsics_weight(), normal_limit); }) @@ -1869,15 +1943,15 @@ mod tests { // given almost full block AllExtrinsicsWeight::put(normal_limit); // will not fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, normal, len).is_err()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); // will fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, op, len).is_ok()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); // likewise for length limit. let len = 100_usize; AllExtrinsicsLen::put(normal_length_limit()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, normal, len).is_err()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, op, len).is_ok()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); }) } @@ -1889,13 +1963,13 @@ mod tests { let len = 0_usize; let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, normal, len) + .validate(&1, CALL, &normal, len) .unwrap() .priority; assert_eq!(priority, 100); let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, op, len) + .validate(&1, CALL, &op, len) .unwrap() .priority; assert_eq!(priority, u64::max_value()); @@ -1913,16 +1987,16 @@ mod tests { if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } }; - reset_check_weight(normal, normal_limit - 1, false); - reset_check_weight(normal, normal_limit, false); - reset_check_weight(normal, normal_limit + 1, true); + reset_check_weight(&normal, normal_limit - 1, false); + reset_check_weight(&normal, normal_limit, false); + reset_check_weight(&normal, normal_limit + 1, true); // Operational ones don't have this limit. let op = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: true }; - reset_check_weight(op, normal_limit, false); - reset_check_weight(op, normal_limit + 100, false); - reset_check_weight(op, 1024, false); - reset_check_weight(op, 1025, true); + reset_check_weight(&op, normal_limit, false); + reset_check_weight(&op, normal_limit + 100, false); + reset_check_weight(&op, 1024, false); + reset_check_weight(&op, 1025, true); }) } @@ -1954,7 +2028,7 @@ mod tests { System::set_block_number(17); >::insert(16, H256::repeat_byte(1)); - assert_eq!(ext.validate(&1, CALL, normal, len).unwrap().longevity, 15); + assert_eq!(ext.validate(&1, CALL, &normal, len).unwrap().longevity, 15); }) } @@ -2013,6 +2087,7 @@ mod tests { let mut ext = new_test_ext(); ext.register_extension(sp_core::traits::CallInWasmExt::new(executor)); ext.execute_with(|| { + System::set_block_number(1); System::set_code( RawOrigin::Root.into(), substrate_test_runtime_client::runtime::WASM_BINARY.to_vec(), @@ -2040,4 +2115,18 @@ mod tests { ).unwrap(); }); } + + #[test] + fn events_not_emitted_during_genesis() { + new_test_ext().execute_with(|| { + // Block Number is zero at genesis + assert!(System::block_number().is_zero()); + System::on_created_account(Default::default()); + assert!(System::events().is_empty()); + // Events will be emitted starting on block 1 + System::set_block_number(1); + System::on_created_account(Default::default()); + assert!(System::events().len() == 1); + }); + } } diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index 5abafe7655172c8b41d81a197d9510730df83f68..a3fe3e00ca4be879ed6074cc6b13f75e0fbae83f 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -95,7 +95,8 @@ impl Signer for TAnyAppPubl } /// Retrieves a public key type for given `SignAndSubmitTransaction`. -pub type PublicOf = < +pub type PublicOf = +< >::CreateTransaction as CreateTransaction>::Extrinsic> @@ -109,7 +110,7 @@ pub type PublicOf = < /// you should use. pub trait SignAndSubmitTransaction { /// Unchecked extrinsic type. - type Extrinsic: ExtrinsicT + codec::Encode; + type Extrinsic: ExtrinsicT + Encode; /// A runtime-specific type to produce signed data for the extrinsic. type CreateTransaction: CreateTransaction; @@ -156,7 +157,7 @@ pub trait SignAndSubmitTransaction { /// you should use. pub trait SubmitUnsignedTransaction { /// Unchecked extrinsic type. - type Extrinsic: ExtrinsicT + codec::Encode; + type Extrinsic: ExtrinsicT + Encode; /// Submit given call to the transaction pool as unsigned transaction. /// @@ -164,7 +165,8 @@ pub trait SubmitUnsignedTransaction { /// and `Err` if transaction was rejected from the pool. fn submit_unsigned(call: impl Into) -> Result<(), ()> { let xt = Self::Extrinsic::new(call.into(), None).ok_or(())?; - sp_io::offchain::submit_transaction(xt.encode()) + let encoded_xt = xt.encode(); + sp_io::offchain::submit_transaction(encoded_xt) } } @@ -291,7 +293,7 @@ impl SignAndSubmitTransaction for TransactionSubmitte T: crate::Trait, C: CreateTransaction, S: Signer<>::Public, >::Signature>, - E: ExtrinsicT + codec::Encode, + E: ExtrinsicT + Encode, { type Extrinsic = E; type CreateTransaction = C; @@ -301,7 +303,7 @@ impl SignAndSubmitTransaction for TransactionSubmitte /// A blanket implementation to use the same submitter for unsigned transactions as well. impl SubmitUnsignedTransaction for TransactionSubmitter where T: crate::Trait, - E: ExtrinsicT + codec::Encode, + E: ExtrinsicT + Encode, { type Extrinsic = E; } @@ -310,7 +312,7 @@ impl SubmitUnsignedTransaction for TransactionSubmitt impl SubmitSignedTransaction for TransactionSubmitter where T: crate::Trait, C: CreateTransaction, - E: ExtrinsicT + codec::Encode, + E: ExtrinsicT + Encode, S: Signer<>::Public, >::Signature>, // Make sure we can unwrap the app crypto key. S: RuntimeAppPublic + AppPublic + Into<::Generic>, diff --git a/frame/timestamp/Cargo.toml b/frame/timestamp/Cargo.toml index ed5e77cb5d1b747ae66280e749103efb933906df..eb7358e197a189dd33b55e985c1b9a65b86a72d8 100644 --- a/frame/timestamp/Cargo.toml +++ b/frame/timestamp/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/pallet-timestamp" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io", optional = true } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } @@ -41,3 +41,6 @@ std = [ "sp-timestamp/std" ] runtime-benchmarks = ["frame-benchmarking", "sp-io"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/timestamp/src/benchmarking.rs b/frame/timestamp/src/benchmarking.rs index 5f69cdbe7e0463d0698b2e9fc12df536c51dc9e9..01a3d502a818e5cca2a166e5fd0a08b1e7ed6b74 100644 --- a/frame/timestamp/src/benchmarking.rs +++ b/frame/timestamp/src/benchmarking.rs @@ -16,10 +16,10 @@ //! Timestamp pallet benchmarking. -use super::*; +#![cfg(feature = "runtime-benchmarks")] +use super::*; use sp_std::prelude::*; - use frame_system::RawOrigin; use frame_benchmarking::benchmarks; @@ -34,3 +34,17 @@ benchmarks! { let n in ...; }: _(RawOrigin::None, n.into()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set::()); + }); + } +} diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index 2a37dfdddb62a89fc7a3aea9f2ce9ef8880421a7..6df8b4606565038d1df8fddce34000bcf3bb8521 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -69,6 +69,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] //! pub fn get_time(origin) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _now = >::get(); @@ -90,20 +91,23 @@ #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(feature = "runtime-benchmarks")] mod benchmarking; use sp_std::{result, cmp}; use sp_inherents::{ProvideInherent, InherentData, InherentIdentifier}; -use frame_support::{Parameter, decl_storage, decl_module}; -use frame_support::traits::{Time, Get}; +#[cfg(feature = "std")] +use frame_support::debug; +use frame_support::{ + Parameter, decl_storage, decl_module, + traits::{Time, UnixTime, Get}, + weights::SimpleDispatchInfo, +}; use sp_runtime::{ RuntimeString, traits::{ AtLeast32Bit, Zero, SaturatedConversion, Scale } }; -use frame_support::weights::SimpleDispatchInfo; use frame_system::ensure_none; use sp_timestamp::{ InherentError, INHERENT_IDENTIFIER, InherentType, @@ -143,7 +147,7 @@ decl_module! { /// `MinimumPeriod`. /// /// The dispatch origin for this call must be `Inherent`. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = SimpleDispatchInfo::FixedMandatory(10_000)] fn set(origin, #[compact] now: T::Moment) { ensure_none(origin)?; assert!(!::DidUpdate::exists(), "Timestamp must be updated only once in the block"); @@ -239,6 +243,25 @@ impl Time for Module { } } +/// Before the timestamp inherent is applied, it returns the time of previous block. +/// +/// On genesis the time returned is not valid. +impl UnixTime for Module { + fn now() -> core::time::Duration { + // now is duration since unix epoch in millisecond as documented in + // `sp_timestamp::InherentDataProvider`. + let now = Self::now(); + sp_std::if_std! { + if now == T::Moment::zero() { + debug::error!( + "`pallet_timestamp::UnixTime::now` is called at genesis, invalid value returned: 0" + ); + } + } + core::time::Duration::from_millis(now.saturated_into::()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -248,6 +271,11 @@ mod tests { use sp_core::H256; use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; + pub fn new_test_ext() -> TestExternalities { + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + TestExternalities::new(t) + } + impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } @@ -293,8 +321,7 @@ mod tests { #[test] fn timestamp_works() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); assert_eq!(Timestamp::now(), 69); @@ -304,8 +331,7 @@ mod tests { #[test] #[should_panic(expected = "Timestamp must be updated only once in the block")] fn double_timestamp_should_fail() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); let _ = Timestamp::dispatch(Call::set(70), Origin::NONE); @@ -315,8 +341,7 @@ mod tests { #[test] #[should_panic(expected = "Timestamp must increment by at least between sequential blocks")] fn block_period_minimum_enforced() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); let _ = Timestamp::dispatch(Call::set(46), Origin::NONE); }); diff --git a/frame/transaction-payment/Cargo.toml b/frame/transaction-payment/Cargo.toml index c9f952d0c4270cd5116adcae850ac8ca329a1492..811c2885b1bdaa674247c7e9510531efa8fe0e95 100644 --- a/frame/transaction-payment/Cargo.toml +++ b/frame/transaction-payment/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage transaction payments" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } @@ -31,3 +31,6 @@ std = [ "frame-system/std", "pallet-transaction-payment-rpc-runtime-api/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/transaction-payment/rpc/Cargo.toml b/frame/transaction-payment/rpc/Cargo.toml index 2e27e530090fe2a8040b0aea5b53b9b745f0d5de..cd74829b29b049fb18b29a661b00bd8763c3c725 100644 --- a/frame/transaction-payment/rpc/Cargo.toml +++ b/frame/transaction-payment/rpc/Cargo.toml @@ -9,9 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "RPC interface for the transaction payment module." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" +jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } sp-rpc = { version = "2.0.0-alpha.5", path = "../../../primitives/rpc" } @@ -20,3 +20,6 @@ sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.5", path = "./runtime-api" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/transaction-payment/rpc/runtime-api/Cargo.toml b/frame/transaction-payment/rpc/runtime-api/Cargo.toml index 02d123203715516da42de9c1fde32edc375e0441..447590111c7ee65ee098a2a508647c5fe7cefc57 100644 --- a/frame/transaction-payment/rpc/runtime-api/Cargo.toml +++ b/frame/transaction-payment/rpc/runtime-api/Cargo.toml @@ -11,7 +11,7 @@ description = "RPC runtime API for transaction payment FRAME pallet" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/runtime" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../support" } @@ -29,3 +29,6 @@ std = [ "sp-runtime/std", "frame-support/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index a9da3830614cce51d82bbfd5d2b9c8ba580313b2..7cf364d700f42142132c8df687ce3ad38884e498 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -44,7 +44,10 @@ use sp_runtime::{ TransactionPriority, ValidTransaction, InvalidTransaction, TransactionValidityError, TransactionValidity, }, - traits::{Zero, Saturating, SignedExtension, SaturatedConversion, Convert}, + traits::{ + Zero, Saturating, SignedExtension, SaturatedConversion, Convert, Dispatchable, + DispatchInfoOf, + }, }; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; @@ -98,7 +101,9 @@ decl_module! { } } -impl Module { +impl Module where + T::Call: Dispatchable, +{ /// Query the data that we know about the fee of a given `call`. /// /// As this module is not and cannot be aware of the internals of a signed extension, it only @@ -106,11 +111,6 @@ impl Module { /// /// All dispatchables must be annotated with weight and will have some fee info. This function /// always returns. - // NOTE: we can actually make it understand `ChargeTransactionPayment`, but would be some hassle - // for sure. We have to make it aware of the index of `ChargeTransactionPayment` in `Extra`. - // Alternatively, we could actually execute the tx's per-dispatch and record the balance of the - // sender before and after the pipeline.. but this is way too much hassle for a very very little - // potential gain in the future. pub fn query_info( unchecked_extrinsic: Extrinsic, len: u32, @@ -119,10 +119,15 @@ impl Module { T: Send + Sync, BalanceOf: Send + Sync, { + // NOTE: we can actually make it understand `ChargeTransactionPayment`, but would be some + // hassle for sure. We have to make it aware of the index of `ChargeTransactionPayment` in + // `Extra`. Alternatively, we could actually execute the tx's per-dispatch and record the + // balance of the sender before and after the pipeline.. but this is way too much hassle for + // a very very little potential gain in the future. let dispatch_info = ::get_dispatch_info(&unchecked_extrinsic); let partial_fee = - >::compute_fee(len, dispatch_info, 0u32.into()); + >::compute_fee(len, &dispatch_info, 0u32.into()); let DispatchInfo { weight, class, .. } = dispatch_info; RuntimeDispatchInfo { weight, class, partial_fee } @@ -134,7 +139,9 @@ impl Module { #[derive(Encode, Decode, Clone, Eq, PartialEq)] pub struct ChargeTransactionPayment(#[codec(compact)] BalanceOf); -impl ChargeTransactionPayment { +impl ChargeTransactionPayment where + T::Call: Dispatchable, +{ /// utility constructor. Used only in client/factory code. pub fn from(fee: BalanceOf) -> Self { Self(fee) @@ -156,7 +163,7 @@ impl ChargeTransactionPayment { /// final_fee = base_fee + targeted_fee_adjustment(len_fee + weight_fee) + tip; pub fn compute_fee( len: u32, - info: ::DispatchInfo, + info: &DispatchInfoOf, tip: BalanceOf, ) -> BalanceOf where @@ -200,14 +207,14 @@ impl sp_std::fmt::Debug for ChargeTransactionPayment } } -impl SignedExtension for ChargeTransactionPayment - where BalanceOf: Send + Sync +impl SignedExtension for ChargeTransactionPayment where + BalanceOf: Send + Sync, + T::Call: Dispatchable, { const IDENTIFIER: &'static str = "ChargeTransactionPayment"; type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } @@ -215,7 +222,7 @@ impl SignedExtension for ChargeTransactionPayment &self, who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { // pay any fees. @@ -438,14 +445,14 @@ mod tests { let len = 10; assert!( ChargeTransactionPayment::::from(0) - .pre_dispatch(&1, CALL, info_from_weight(5), len) + .pre_dispatch(&1, CALL, &info_from_weight(5), len) .is_ok() ); assert_eq!(Balances::free_balance(1), 100 - 5 - 5 - 10); assert!( ChargeTransactionPayment::::from(5 /* tipped */) - .pre_dispatch(&2, CALL, info_from_weight(3), len) + .pre_dispatch(&2, CALL, &info_from_weight(3), len) .is_ok() ); assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 3 - 5); @@ -463,7 +470,7 @@ mod tests { // maximum weight possible assert!( ChargeTransactionPayment::::from(0) - .pre_dispatch(&1, CALL, info_from_weight(Weight::max_value()), 10) + .pre_dispatch(&1, CALL, &info_from_weight(Weight::max_value()), 10) .is_ok() ); // fee will be proportional to what is the actual maximum weight in the runtime. @@ -495,7 +502,7 @@ mod tests { }; assert!( ChargeTransactionPayment::::from(0) - .validate(&1, CALL, operational_transaction , len) + .validate(&1, CALL, &operational_transaction , len) .is_ok() ); @@ -507,7 +514,7 @@ mod tests { }; assert!( ChargeTransactionPayment::::from(0) - .validate(&1, CALL, free_transaction , len) + .validate(&1, CALL, &free_transaction , len) .is_err() ); }); @@ -527,7 +534,7 @@ mod tests { assert!( ChargeTransactionPayment::::from(10) // tipped - .pre_dispatch(&1, CALL, info_from_weight(3), len) + .pre_dispatch(&1, CALL, &info_from_weight(3), len) .is_ok() ); assert_eq!(Balances::free_balance(1), 100 - 10 - 5 - (10 + 3) * 3 / 2); @@ -587,25 +594,25 @@ mod tests { class: DispatchClass::Operational, pays_fee: false, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 10), 10); + assert_eq!(ChargeTransactionPayment::::compute_fee(0, &dispatch_info, 10), 10); // No tip, only base fee works let dispatch_info = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: true, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 100); + assert_eq!(ChargeTransactionPayment::::compute_fee(0, &dispatch_info, 0), 100); // Tip + base fee works - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 69), 169); + assert_eq!(ChargeTransactionPayment::::compute_fee(0, &dispatch_info, 69), 169); // Len (byte fee) + base fee works - assert_eq!(ChargeTransactionPayment::::compute_fee(42, dispatch_info, 0), 520); + assert_eq!(ChargeTransactionPayment::::compute_fee(42, &dispatch_info, 0), 520); // Weight fee + base fee works let dispatch_info = DispatchInfo { weight: 1000, class: DispatchClass::Operational, pays_fee: true, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 1100); + assert_eq!(ChargeTransactionPayment::::compute_fee(0, &dispatch_info, 0), 1100); }); } @@ -626,7 +633,7 @@ mod tests { class: DispatchClass::Operational, pays_fee: true, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 100); + assert_eq!(ChargeTransactionPayment::::compute_fee(0, &dispatch_info, 0), 100); // Everything works together :) let dispatch_info = DispatchInfo { @@ -638,7 +645,7 @@ mod tests { // adjustable fee = (123 * 1) + (456 * 10) = 4683 // adjusted fee = (4683 * .5) + 4683 = 7024.5 -> 7024 // final fee = 100 + 7024 + 789 tip = 7913 - assert_eq!(ChargeTransactionPayment::::compute_fee(456, dispatch_info, 789), 7913); + assert_eq!(ChargeTransactionPayment::::compute_fee(456, &dispatch_info, 789), 7913); }); } @@ -660,7 +667,7 @@ mod tests { assert_eq!( ChargeTransactionPayment::::compute_fee( ::max_value(), - dispatch_info, + &dispatch_info, ::max_value() ), ::max_value() diff --git a/frame/treasury/Cargo.toml b/frame/treasury/Cargo.toml index a673c7ce81740b812d0af8f31f39d63735b7addb..6951c0eba9ae7bd566681e385ab8bf7ca9400718 100644 --- a/frame/treasury/Cargo.toml +++ b/frame/treasury/Cargo.toml @@ -10,14 +10,14 @@ description = "FRAME pallet to manage treasury" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } -frame-benchmarking = { version = "2.0.0-alpha.2", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } @@ -38,3 +38,6 @@ runtime-benchmarks = [ "frame-benchmarking", "frame-support/runtime-benchmarks", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/treasury/src/benchmarking.rs b/frame/treasury/src/benchmarking.rs index fb16af740a8a3765c509f3a39a1641ff0bdff205..f901576c95d4b0c8d384737502654672ccd8a795 100644 --- a/frame/treasury/src/benchmarking.rs +++ b/frame/treasury/src/benchmarking.rs @@ -16,11 +16,13 @@ //! Treasury pallet benchmarking. +#![cfg(feature = "runtime-benchmarks")] + use super::*; use frame_system::RawOrigin; use frame_benchmarking::{benchmarks, account}; -use sp_runtime::traits::OnInitialize; +use frame_support::traits::OnInitialize; use crate::Module as Treasury; @@ -56,12 +58,15 @@ fn setup_awesome(length: u32) -> (T::AccountId, Vec, T::AccountId) fn setup_tip(r: u32, t: u32) -> Result<(T::AccountId, Vec, T::AccountId, BalanceOf), &'static str> { + let tippers_count = T::Tippers::count(); + for i in 0 .. t { let member = account("member", i, SEED); T::Tippers::add(&member); + ensure!(T::Tippers::contains(&member), "failed to add tipper"); } - ensure!(T::Tippers::count() == t as usize, "problem creating tippers"); + ensure!(T::Tippers::count() == tippers_count + t as usize, "problem creating tippers"); let caller = account("member", t - 1, SEED); let reason = vec![0; r as usize]; let beneficiary = account("beneficiary", t, SEED); @@ -69,16 +74,19 @@ fn setup_tip(r: u32, t: u32) -> Ok((caller, reason, beneficiary, value)) } -// Create `t` new types for the tip proposal with `hash`. -// This function automatically moves forward the block number to a time which -// would resolve the tipping process. +// Create `t` new tips for the tip proposal with `hash`. +// This function automatically makes the tip able to close. fn create_tips(t: u32, hash: T::Hash, value: BalanceOf) -> Result<(), &'static str> { for i in 0 .. t { let caller = account("member", i, SEED); ensure!(T::Tippers::contains(&caller), "caller is not a tipper"); Treasury::::tip(RawOrigin::Signed(caller).into(), hash, value)?; } - frame_system::Module::::set_block_number(T::TipCountdown::get() * 10.into()); + Tips::::mutate(hash, |maybe_tip| { + if let Some(open_tip) = maybe_tip { + open_tip.closes = Some(T::BlockNumber::zero()); + } + }); Ok(()) } @@ -209,3 +217,25 @@ benchmarks! { Treasury::::on_initialize(T::BlockNumber::zero()); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_propose_spend::()); + assert_ok!(test_benchmark_reject_proposal::()); + assert_ok!(test_benchmark_approve_proposal::()); + assert_ok!(test_benchmark_report_awesome::()); + assert_ok!(test_benchmark_retract_tip::()); + assert_ok!(test_benchmark_tip_new::()); + assert_ok!(test_benchmark_tip::()); + assert_ok!(test_benchmark_close_tip::()); + assert_ok!(test_benchmark_on_initialize::()); + }); + } +} diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index 2c808e065984e51435f3f208a2edcc66d34538ca..f07b9b511e1682edfd3c450239ed95eef2f546a2 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -96,15 +96,14 @@ use frame_support::traits::{ ReservableCurrency, WithdrawReason }; use sp_runtime::{Permill, ModuleId, Percent, RuntimeDebug, traits::{ - Zero, EnsureOrigin, StaticLookup, AccountIdConversion, Saturating, Hash, BadOrigin + Zero, StaticLookup, AccountIdConversion, Saturating, Hash, BadOrigin }}; -use frame_support::{weights::SimpleDispatchInfo, traits::Contains}; +use frame_support::weights::{Weight, WeighData, SimpleDispatchInfo}; +use frame_support::traits::{Contains, EnsureOrigin}; use codec::{Encode, Decode}; use frame_system::{self as system, ensure_signed, ensure_root}; -#[cfg(test)] mod tests; -#[cfg(feature = "runtime-benchmarks")] mod benchmarking; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -553,11 +552,13 @@ decl_module! { Self::payout_tip(tip); } - fn on_initialize(n: T::BlockNumber) { + fn on_initialize(n: T::BlockNumber) -> Weight { // Check to see if we should spend some funds! if (n % T::SpendPeriod::get()).is_zero() { Self::spend_funds(); } + + SimpleDispatchInfo::default().weigh_data(()) } } } diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index d7e710639dccff45d14e08fb021fb70eb544b9fd..5ad78dcad79356ce57eb59305cdf3a96d65682c6 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -1,12 +1,34 @@ -use super::*; +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Treasury pallet tests. + +#![cfg(test)] -use frame_support::{assert_noop, assert_ok, impl_outer_origin, parameter_types, weights::Weight}; -use frame_support::traits::Contains; +use super::*; +use std::cell::RefCell; +use frame_support::{ + assert_noop, assert_ok, impl_outer_origin, parameter_types, weights::Weight, + traits::{Contains, OnInitialize} +}; use sp_core::H256; use sp_runtime::{ Perbill, testing::Header, - traits::{BlakeTwo256, OnInitialize, IdentityLookup, BadOrigin}, + traits::{BlakeTwo256, IdentityLookup, BadOrigin}, }; impl_outer_origin! { @@ -52,16 +74,24 @@ impl pallet_balances::Trait for Test { type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; } +thread_local! { + static TEN_TO_FOURTEEN: RefCell> = RefCell::new(vec![10,11,12,13,14]); +} pub struct TenToFourteen; impl Contains for TenToFourteen { - fn contains(n: &u64) -> bool { - *n >= 10 && *n <= 14 - } fn sorted_members() -> Vec { - vec![10, 11, 12, 13, 14] + TEN_TO_FOURTEEN.with(|v| { + v.borrow().clone() + }) } #[cfg(feature = "runtime-benchmarks")] - fn add(_: &u64) { unimplemented!() } + fn add(new: &u64) { + TEN_TO_FOURTEEN.with(|v| { + let mut members = v.borrow_mut(); + members.push(*new); + members.sort(); + }) + } } parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); @@ -93,7 +123,7 @@ type System = frame_system::Module; type Balances = pallet_balances::Module; type Treasury = Module; -fn new_test_ext() -> sp_io::TestExternalities { +pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig::{ // Total issuance will be 200 with treasury account initialized at ED. diff --git a/frame/utility/Cargo.toml b/frame/utility/Cargo.toml index 690465b5d2219ab3bc1cafb77b423b0a863488e8..cf1042d8521da3e0c8c538eaef51c48de03b5439 100644 --- a/frame/utility/Cargo.toml +++ b/frame/utility/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME utilities pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } @@ -18,6 +18,8 @@ sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../. sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } + [dev-dependencies] sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } @@ -33,3 +35,10 @@ std = [ "sp-io/std", "sp-std/std" ] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", +] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/utility/src/benchmarking.rs b/frame/utility/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..fc8783b49aa87aaf0813bcc467defae82abc7bf3 --- /dev/null +++ b/frame/utility/src/benchmarking.rs @@ -0,0 +1,169 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// Benchmarks for Utility Pallet + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Saturating; + +use crate::Module as Utility; + +const SEED: u32 = 0; + +fn setup_multi(s: u32, z: u32) -> Result<(Vec, Box<::Call>), &'static str>{ + let mut signatories: Vec = Vec::new(); + for i in 0 .. s { + let signatory = account("signatory", i, SEED); + // Give them some balance for a possible deposit + let deposit = T::MultisigDepositBase::get() + T::MultisigDepositFactor::get() * s.into(); + let balance = T::Currency::minimum_balance().saturating_mul(100.into()) + deposit; + T::Currency::make_free_balance_be(&signatory, balance); + signatories.push(signatory); + } + signatories.sort(); + let call: Box<::Call> = Box::new(frame_system::Call::remark(vec![0; z as usize]).into()); + return Ok((signatories, call)) +} + +benchmarks! { + _ { } + + batch { + let c in 0 .. 1000; + let mut calls: Vec<::Call> = Vec::new(); + for i in 0 .. c { + let call = frame_system::Call::remark(vec![]).into(); + calls.push(call); + } + let caller = account("caller", 0, SEED); + }: _(RawOrigin::Signed(caller), calls) + + as_sub { + let u in 0 .. 1000; + let caller = account("caller", u, SEED); + let call = Box::new(frame_system::Call::remark(vec![]).into()); + }: _(RawOrigin::Signed(caller), u as u16, call) + + as_multi_create { + // Signatories, need at least 2 total people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + }: as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call) + + as_multi_approve { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + // before the call, get the timepoint + let timepoint = Utility::::timepoint(); + // Create the multi + Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; + let caller2 = signatories2.remove(0); + }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call) + + as_multi_complete { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + // before the call, get the timepoint + let timepoint = Utility::::timepoint(); + // Create the multi + Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; + // Everyone except the first person approves + for i in 1 .. s - 1 { + let mut signatories_loop = signatories2.clone(); + let caller_loop = signatories_loop.remove(i as usize); + Utility::::as_multi(RawOrigin::Signed(caller_loop).into(), s as u16, signatories_loop, Some(timepoint), call.clone())?; + } + let caller2 = signatories2.remove(0); + }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call) + + approve_as_multi_create { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = call.using_encoded(blake2_256); + // Create the multi + }: approve_as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call_hash) + + approve_as_multi_approve { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = call.using_encoded(blake2_256); + // before the call, get the timepoint + let timepoint = Utility::::timepoint(); + // Create the multi + Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; + let caller2 = signatories2.remove(0); + }: approve_as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call_hash) + + cancel_as_multi { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = call.using_encoded(blake2_256); + let timepoint = Utility::::timepoint(); + // Create the multi + Utility::::as_multi(RawOrigin::Signed(caller.clone()).into(), s as u16, signatories.clone(), None, call.clone())?; + }: _(RawOrigin::Signed(caller), s as u16, signatories, timepoint, call_hash) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_batch::()); + assert_ok!(test_benchmark_as_sub::()); + assert_ok!(test_benchmark_as_multi_create::()); + assert_ok!(test_benchmark_as_multi_approve::()); + assert_ok!(test_benchmark_as_multi_complete::()); + assert_ok!(test_benchmark_approve_as_multi_create::()); + assert_ok!(test_benchmark_approve_as_multi_approve::()); + assert_ok!(test_benchmark_cancel_as_multi::()); + }); + } +} diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs index 0b60532c3dd23c35d09ff09717eb79d3217ef212..3cd6d103cc0940e4ab75d037c8f703ef8a0560d5 100644 --- a/frame/utility/src/lib.rs +++ b/frame/utility/src/lib.rs @@ -68,10 +68,14 @@ use sp_io::hashing::blake2_256; use frame_support::{decl_module, decl_event, decl_error, decl_storage, Parameter, ensure, RuntimeDebug}; use frame_support::{traits::{Get, ReservableCurrency, Currency}, weights::{GetDispatchInfo, DispatchClass,FunctionOf}, + dispatch::PostDispatchInfo, }; use frame_system::{self as system, ensure_signed}; use sp_runtime::{DispatchError, DispatchResult, traits::Dispatchable}; +mod tests; +mod benchmarking; + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; /// Configuration trait. @@ -80,7 +84,7 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// The overarching call type. - type Call: Parameter + Dispatchable + GetDispatchInfo; + type Call: Parameter + Dispatchable + GetDispatchInfo + From>; /// The currency mechanism. type Currency: ReservableCurrency; @@ -166,7 +170,8 @@ decl_event! { /// Events type. pub enum Event where AccountId = ::AccountId, - BlockNumber = ::BlockNumber + BlockNumber = ::BlockNumber, + CallHash = [u8; 32] { /// Batch of dispatches did not complete fully. Index of first failing dispatch given, as /// well as the error. @@ -174,17 +179,17 @@ decl_event! { /// Batch of dispatches completed fully with no error. BatchCompleted, /// A new multisig operation has begun. First param is the account that is approving, - /// second is the multisig account. - NewMultisig(AccountId, AccountId), + /// second is the multisig account, third is hash of the call. + NewMultisig(AccountId, AccountId, CallHash), /// A multisig operation has been approved by someone. First param is the account that is - /// approving, third is the multisig account. - MultisigApproval(AccountId, Timepoint, AccountId), + /// approving, third is the multisig account, fourth is hash of the call. + MultisigApproval(AccountId, Timepoint, AccountId, CallHash), /// A multisig operation has been executed. First param is the account that is - /// approving, third is the multisig account. - MultisigExecuted(AccountId, Timepoint, AccountId, DispatchResult), + /// approving, third is the multisig account, fourth is hash of the call to be executed. + MultisigExecuted(AccountId, Timepoint, AccountId, CallHash, DispatchResult), /// A multisig operation has been cancelled. First param is the account that is - /// cancelling, third is the multisig account. - MultisigCancelled(AccountId, Timepoint, AccountId), + /// cancelling, third is the multisig account, fourth is hash of the call. + MultisigCancelled(AccountId, Timepoint, AccountId, CallHash), } } @@ -243,7 +248,7 @@ decl_module! { for (index, call) in calls.into_iter().enumerate() { let result = call.dispatch(origin.clone()); if let Err(e) = result { - Self::deposit_event(Event::::BatchInterrupted(index as u32, e)); + Self::deposit_event(Event::::BatchInterrupted(index as u32, e.error)); return Ok(()); } } @@ -266,6 +271,7 @@ decl_module! { let who = ensure_signed(origin)?; let pseudonym = Self::sub_account_id(who, index); call.dispatch(frame_system::RawOrigin::Signed(pseudonym).into()) + .map(|_| ()).map_err(|e| e.error) } /// Register approval for a dispatch to be made from a deterministic composite account if @@ -342,7 +348,7 @@ decl_module! { if (m.approvals.len() as u16) < threshold - 1 { m.approvals.insert(pos, who.clone()); >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id)); + Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); return Ok(()) } } else { @@ -354,7 +360,9 @@ decl_module! { let result = call.dispatch(frame_system::RawOrigin::Signed(id.clone()).into()); let _ = T::Currency::unreserve(&m.depositor, m.deposit); >::remove(&id, call_hash); - Self::deposit_event(RawEvent::MultisigExecuted(who, timepoint, id, result)); + Self::deposit_event(RawEvent::MultisigExecuted( + who, timepoint, id, call_hash, result.map(|_| ()).map_err(|e| e.error) + )); } else { ensure!(maybe_timepoint.is_none(), Error::::UnexpectedTimepoint); if threshold > 1 { @@ -367,9 +375,10 @@ decl_module! { depositor: who.clone(), approvals: vec![who.clone()], }); - Self::deposit_event(RawEvent::NewMultisig(who, id)); + Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); } else { return call.dispatch(frame_system::RawOrigin::Signed(id).into()) + .map(|_| ()).map_err(|e| e.error) } } Ok(()) @@ -436,7 +445,7 @@ decl_module! { if let Err(pos) = m.approvals.binary_search(&who) { m.approvals.insert(pos, who.clone()); >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id)); + Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); } else { Err(Error::::AlreadyApproved)? } @@ -452,7 +461,7 @@ decl_module! { depositor: who.clone(), approvals: vec![who.clone()], }); - Self::deposit_event(RawEvent::NewMultisig(who, id)); + Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); } else { Err(Error::::NoApprovalsNeeded)? } @@ -512,7 +521,7 @@ decl_module! { let _ = T::Currency::unreserve(&m.depositor, m.deposit); >::remove(&id, call_hash); - Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id)); + Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id, call_hash)); Ok(()) } } @@ -563,426 +572,3 @@ impl Module { Ok(signatories) } } - -#[cfg(test)] -mod tests { - use super::*; - - use frame_support::{ - assert_ok, assert_noop, impl_outer_origin, parameter_types, impl_outer_dispatch, - weights::Weight, impl_outer_event - }; - use sp_core::H256; - use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; - use crate as utility; - - impl_outer_origin! { - pub enum Origin for Test where system = frame_system {} - } - - impl_outer_event! { - pub enum TestEvent for Test { - system, - pallet_balances, - utility, - } - } - impl_outer_dispatch! { - pub enum Call for Test where origin: Origin { - pallet_balances::Balances, - utility::Utility, - } - } - - // For testing the pallet, we construct most of a mock runtime. This means - // first constructing a configuration type (`Test`) which `impl`s each of the - // configuration traits of pallets we want to use. - #[derive(Clone, Eq, PartialEq)] - pub struct Test; - parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const MaximumBlockWeight: Weight = 1024; - pub const MaximumBlockLength: u32 = 2 * 1024; - pub const AvailableBlockRatio: Perbill = Perbill::one(); - } - impl frame_system::Trait for Test { - type Origin = Origin; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Call = Call; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = TestEvent; - type BlockHashCount = BlockHashCount; - type MaximumBlockWeight = MaximumBlockWeight; - type MaximumBlockLength = MaximumBlockLength; - type AvailableBlockRatio = AvailableBlockRatio; - type Version = (); - type ModuleToIndex = (); - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - } - parameter_types! { - pub const ExistentialDeposit: u64 = 1; - } - impl pallet_balances::Trait for Test { - type Balance = u64; - type Event = TestEvent; - type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - } - parameter_types! { - pub const MultisigDepositBase: u64 = 1; - pub const MultisigDepositFactor: u64 = 1; - pub const MaxSignatories: u16 = 3; - } - impl Trait for Test { - type Event = TestEvent; - type Call = Call; - type Currency = Balances; - type MultisigDepositBase = MultisigDepositBase; - type MultisigDepositFactor = MultisigDepositFactor; - type MaxSignatories = MaxSignatories; - } - type System = frame_system::Module; - type Balances = pallet_balances::Module; - type Utility = Module; - - use pallet_balances::Call as BalancesCall; - use pallet_balances::Error as BalancesError; - - fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - pallet_balances::GenesisConfig:: { - balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 10)], - }.assimilate_storage(&mut t).unwrap(); - t.into() - } - - fn last_event() -> TestEvent { - system::Module::::events().pop().map(|e| e.event).expect("Event expected") - } - - fn expect_event>(e: E) { - assert_eq!(last_event(), e.into()); - } - - fn now() -> Timepoint { - Utility::timepoint() - } - - #[test] - fn multisig_deposit_is_taken_and_returned() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_eq!(Balances::free_balance(1), 2); - assert_eq!(Balances::reserved_balance(1), 3); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(1), 5); - assert_eq!(Balances::reserved_balance(1), 0); - }); - } - - #[test] - fn cancel_multisig_returns_deposit() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_eq!(Balances::free_balance(1), 6); - assert_eq!(Balances::reserved_balance(1), 4); - assert_ok!( - Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), - ); - assert_eq!(Balances::free_balance(1), 10); - assert_eq!(Balances::reserved_balance(1), 0); - }); - } - - #[test] - fn timepoint_checking_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - - assert_noop!( - Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone()), - Error::::UnexpectedTimepoint, - ); - - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); - - assert_noop!( - Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call.clone()), - Error::::NoTimepoint, - ); - let later = Timepoint { index: 1, .. now() }; - assert_noop!( - Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(later), call.clone()), - Error::::WrongTimepoint, - ); - }); - } - - #[test] - fn multisig_2_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); - } - - #[test] - fn multisig_3_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 3); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); - } - - #[test] - fn cancel_multisig_works() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_noop!( - Utility::cancel_as_multi(Origin::signed(2), 3, vec![1, 3], now(), hash.clone()), - Error::::NotOwner, - ); - assert_ok!( - Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), - ); - }); - } - - #[test] - fn multisig_2_of_3_as_multi_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); - } - - #[test] - fn multisig_2_of_3_as_multi_with_many_calls_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call1 = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); - let call2 = Box::new(Call::Balances(BalancesCall::transfer(7, 5))); - - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call1.clone())); - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call2.clone())); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call2)); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call1)); - - assert_eq!(Balances::free_balance(6), 10); - assert_eq!(Balances::free_balance(7), 5); - }); - } - - #[test] - fn multisig_2_of_3_cannot_reissue_same_call() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call.clone())); - assert_eq!(Balances::free_balance(multi), 5); - - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call)); - - let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); - expect_event(RawEvent::MultisigExecuted(3, now(), multi, Err(err))); - }); - } - - #[test] - fn zero_threshold_fails() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_noop!( - Utility::as_multi(Origin::signed(1), 0, vec![2], None, call), - Error::::ZeroThreshold, - ); - }); - } - - #[test] - fn too_many_signatories_fails() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_noop!( - Utility::as_multi(Origin::signed(1), 2, vec![2, 3, 4], None, call.clone()), - Error::::TooManySignatories, - ); - }); - } - - #[test] - fn duplicate_approvals_are_ignored() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash.clone())); - assert_noop!( - Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], Some(now()), hash.clone()), - Error::::AlreadyApproved, - ); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone())); - assert_noop!( - Utility::approve_as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), hash.clone()), - Error::::NoApprovalsNeeded, - ); - }); - } - - #[test] - fn multisig_1_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 1); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_noop!( - Utility::approve_as_multi(Origin::signed(1), 1, vec![2, 3], None, hash.clone()), - Error::::NoApprovalsNeeded, - ); - assert_noop!( - Utility::as_multi(Origin::signed(4), 1, vec![2, 3], None, call.clone()), - BalancesError::::InsufficientBalance, - ); - assert_ok!(Utility::as_multi(Origin::signed(1), 1, vec![2, 3], None, call)); - - assert_eq!(Balances::free_balance(6), 15); - }); - } - - #[test] - fn as_sub_works() { - new_test_ext().execute_with(|| { - let sub_1_0 = Utility::sub_account_id(1, 0); - assert_ok!(Balances::transfer(Origin::signed(1), sub_1_0, 5)); - assert_noop!(Utility::as_sub( - Origin::signed(1), - 1, - Box::new(Call::Balances(BalancesCall::transfer(6, 3))), - ), BalancesError::::InsufficientBalance); - assert_ok!(Utility::as_sub( - Origin::signed(1), - 0, - Box::new(Call::Balances(BalancesCall::transfer(2, 3))), - )); - assert_eq!(Balances::free_balance(sub_1_0), 2); - assert_eq!(Balances::free_balance(2), 13); - }); - } - - #[test] - fn batch_with_root_works() { - new_test_ext().execute_with(|| { - assert_eq!(Balances::free_balance(1), 10); - assert_eq!(Balances::free_balance(2), 10); - assert_ok!(Utility::batch(Origin::ROOT, vec![ - Call::Balances(BalancesCall::force_transfer(1, 2, 5)), - Call::Balances(BalancesCall::force_transfer(1, 2, 5)) - ])); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::free_balance(2), 20); - }); - } - - #[test] - fn batch_with_signed_works() { - new_test_ext().execute_with(|| { - assert_eq!(Balances::free_balance(1), 10); - assert_eq!(Balances::free_balance(2), 10); - assert_ok!( - Utility::batch(Origin::signed(1), vec![ - Call::Balances(BalancesCall::transfer(2, 5)), - Call::Balances(BalancesCall::transfer(2, 5)) - ]), - ); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::free_balance(2), 20); - }); - } - - #[test] - fn batch_early_exit_works() { - new_test_ext().execute_with(|| { - assert_eq!(Balances::free_balance(1), 10); - assert_eq!(Balances::free_balance(2), 10); - assert_ok!( - Utility::batch(Origin::signed(1), vec![ - Call::Balances(BalancesCall::transfer(2, 5)), - Call::Balances(BalancesCall::transfer(2, 10)), - Call::Balances(BalancesCall::transfer(2, 5)), - ]), - ); - assert_eq!(Balances::free_balance(1), 5); - assert_eq!(Balances::free_balance(2), 15); - }); - } -} diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..68bdabd6d9b7fe418bad496305a603f61551fbf7 --- /dev/null +++ b/frame/utility/src/tests.rs @@ -0,0 +1,442 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +// Tests for Utility Pallet + +#![cfg(test)] + +use super::*; + +use frame_support::{ + assert_ok, assert_noop, impl_outer_origin, parameter_types, impl_outer_dispatch, + weights::Weight, impl_outer_event +}; +use sp_core::H256; +use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; +use crate as utility; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +impl_outer_event! { + pub enum TestEvent for Test { + system, + pallet_balances, + utility, + } +} +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + frame_system::System, + pallet_balances::Balances, + utility::Utility, + } +} + +// For testing the pallet, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of pallets we want to use. +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = TestEvent; + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); +} +parameter_types! { + pub const ExistentialDeposit: u64 = 1; +} +impl pallet_balances::Trait for Test { + type Balance = u64; + type Event = TestEvent; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} +parameter_types! { + pub const MultisigDepositBase: u64 = 1; + pub const MultisigDepositFactor: u64 = 1; + pub const MaxSignatories: u16 = 3; +} +impl Trait for Test { + type Event = TestEvent; + type Call = Call; + type Currency = Balances; + type MultisigDepositBase = MultisigDepositBase; + type MultisigDepositFactor = MultisigDepositFactor; + type MaxSignatories = MaxSignatories; +} +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Utility = Module; + +use pallet_balances::Call as BalancesCall; +use pallet_balances::Error as BalancesError; + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 10)], + }.assimilate_storage(&mut t).unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +fn last_event() -> TestEvent { + system::Module::::events().pop().map(|e| e.event).expect("Event expected") +} + +fn expect_event>(e: E) { + assert_eq!(last_event(), e.into()); +} + +fn now() -> Timepoint { + Utility::timepoint() +} + +#[test] +fn multisig_deposit_is_taken_and_returned() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); + assert_eq!(Balances::free_balance(1), 2); + assert_eq!(Balances::reserved_balance(1), 3); + + assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); + assert_eq!(Balances::free_balance(1), 5); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn cancel_multisig_returns_deposit() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); + assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); + assert_eq!(Balances::free_balance(1), 6); + assert_eq!(Balances::reserved_balance(1), 4); + assert_ok!( + Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), + ); + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn timepoint_checking_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + + assert_noop!( + Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone()), + Error::::UnexpectedTimepoint, + ); + + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); + + assert_noop!( + Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call.clone()), + Error::::NoTimepoint, + ); + let later = Timepoint { index: 1, .. now() }; + assert_noop!( + Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(later), call.clone()), + Error::::WrongTimepoint, + ); + }); +} + +#[test] +fn multisig_2_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_3_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 3); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); + assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Utility::as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), call)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn cancel_multisig_works() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); + assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); + assert_noop!( + Utility::cancel_as_multi(Origin::signed(2), 3, vec![1, 3], now(), hash.clone()), + Error::::NotOwner, + ); + assert_ok!( + Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), + ); + }); +} + +#[test] +fn multisig_2_of_3_as_multi_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_2_of_3_as_multi_with_many_calls_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call1 = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); + let call2 = Box::new(Call::Balances(BalancesCall::transfer(7, 5))); + + assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call1.clone())); + assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call2.clone())); + assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call2)); + assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call1)); + + assert_eq!(Balances::free_balance(6), 10); + assert_eq!(Balances::free_balance(7), 5); + }); +} + +#[test] +fn multisig_2_of_3_cannot_reissue_same_call() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); + assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); + assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call.clone())); + assert_eq!(Balances::free_balance(multi), 5); + + assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); + assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call.clone())); + + let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); + expect_event(RawEvent::MultisigExecuted(3, now(), multi, call.using_encoded(blake2_256), Err(err))); + }); +} + +#[test] +fn zero_threshold_fails() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + assert_noop!( + Utility::as_multi(Origin::signed(1), 0, vec![2], None, call), + Error::::ZeroThreshold, + ); + }); +} + +#[test] +fn too_many_signatories_fails() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + assert_noop!( + Utility::as_multi(Origin::signed(1), 2, vec![2, 3, 4], None, call.clone()), + Error::::TooManySignatories, + ); + }); +} + +#[test] +fn duplicate_approvals_are_ignored() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash.clone())); + assert_noop!( + Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], Some(now()), hash.clone()), + Error::::AlreadyApproved, + ); + assert_ok!(Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone())); + assert_noop!( + Utility::approve_as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), hash.clone()), + Error::::NoApprovalsNeeded, + ); + }); +} + +#[test] +fn multisig_1_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Utility::multi_account_id(&[1, 2, 3][..], 1); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + let hash = call.using_encoded(blake2_256); + assert_noop!( + Utility::approve_as_multi(Origin::signed(1), 1, vec![2, 3], None, hash.clone()), + Error::::NoApprovalsNeeded, + ); + assert_noop!( + Utility::as_multi(Origin::signed(4), 1, vec![2, 3], None, call.clone()), + BalancesError::::InsufficientBalance, + ); + assert_ok!(Utility::as_multi(Origin::signed(1), 1, vec![2, 3], None, call)); + + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn as_sub_works() { + new_test_ext().execute_with(|| { + let sub_1_0 = Utility::sub_account_id(1, 0); + assert_ok!(Balances::transfer(Origin::signed(1), sub_1_0, 5)); + assert_noop!(Utility::as_sub( + Origin::signed(1), + 1, + Box::new(Call::Balances(BalancesCall::transfer(6, 3))), + ), BalancesError::::InsufficientBalance); + assert_ok!(Utility::as_sub( + Origin::signed(1), + 0, + Box::new(Call::Balances(BalancesCall::transfer(2, 3))), + )); + assert_eq!(Balances::free_balance(sub_1_0), 2); + assert_eq!(Balances::free_balance(2), 13); + }); +} + +#[test] +fn batch_with_root_works() { + new_test_ext().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::free_balance(2), 10); + assert_ok!(Utility::batch(Origin::ROOT, vec![ + Call::Balances(BalancesCall::force_transfer(1, 2, 5)), + Call::Balances(BalancesCall::force_transfer(1, 2, 5)) + ])); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::free_balance(2), 20); + }); +} + +#[test] +fn batch_with_signed_works() { + new_test_ext().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::free_balance(2), 10); + assert_ok!( + Utility::batch(Origin::signed(1), vec![ + Call::Balances(BalancesCall::transfer(2, 5)), + Call::Balances(BalancesCall::transfer(2, 5)) + ]), + ); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::free_balance(2), 20); + }); +} + +#[test] +fn batch_early_exit_works() { + new_test_ext().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::free_balance(2), 10); + assert_ok!( + Utility::batch(Origin::signed(1), vec![ + Call::Balances(BalancesCall::transfer(2, 5)), + Call::Balances(BalancesCall::transfer(2, 10)), + Call::Balances(BalancesCall::transfer(2, 5)), + ]), + ); + assert_eq!(Balances::free_balance(1), 5); + assert_eq!(Balances::free_balance(2), 15); + }); +} diff --git a/frame/vesting/Cargo.toml b/frame/vesting/Cargo.toml index c8d46855c5e0f18bf18a94d4eeb6b8ccdc72a08f..e40062706ff8559101e0390300b2a8af58a4174c 100644 --- a/frame/vesting/Cargo.toml +++ b/frame/vesting/Cargo.toml @@ -10,7 +10,7 @@ description = "FRAME pallet for manage vesting" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -36,4 +36,7 @@ std = [ "frame-support/std", "frame-system/std", ] -runtime-benchmarks = ["frame-benchmarking", "frame-system/runtime-benchmarks"] +runtime-benchmarks = ["frame-benchmarking"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/vesting/src/benchmarking.rs b/frame/vesting/src/benchmarking.rs index 8d0f0214eb293d8fbac722dd034462997cc1cacc..be2cb4cb2b0aad0a2fe1578a3f440dc101225811 100644 --- a/frame/vesting/src/benchmarking.rs +++ b/frame/vesting/src/benchmarking.rs @@ -16,6 +16,8 @@ //! Vesting pallet benchmarking. +#![cfg(feature = "runtime-benchmarks")] + use super::*; use frame_system::{RawOrigin, Module as System}; @@ -119,6 +121,24 @@ benchmarks! { }; let _ = T::Currency::make_free_balance_be(&from, transfer_amount * 10.into()); - + }: _(RawOrigin::Signed(from), to_lookup, vesting_schedule) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{ExtBuilder, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + ExtBuilder::default().existential_deposit(256).build().execute_with(|| { + assert_ok!(test_benchmark_vest_locked::()); + assert_ok!(test_benchmark_vest_not_locked::()); + assert_ok!(test_benchmark_vest_other_locked::()); + assert_ok!(test_benchmark_vest_other_not_locked::()); + assert_ok!(test_benchmark_vested_transfer::()); + }); + } +} diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs index 94a70b7ef132f814a0015b69a6ba31e4513a047a..b0c98e78bd6f7243e4c017f133726fccf16407a9 100644 --- a/frame/vesting/src/lib.rs +++ b/frame/vesting/src/lib.rs @@ -60,7 +60,6 @@ use frame_support::traits::{ use frame_support::weights::SimpleDispatchInfo; use frame_system::{self as system, ensure_signed}; -#[cfg(feature = "runtime-benchmarks")] mod benchmarking; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -195,6 +194,7 @@ decl_module! { /// - One storage read (codec `O(1)`) and up to one removal. /// - One event. /// # + #[weight = SimpleDispatchInfo::default()] fn vest(origin) -> DispatchResult { let who = ensure_signed(origin)?; Self::update_lock(who) @@ -216,6 +216,7 @@ decl_module! { /// - One storage read (codec `O(1)`) and up to one removal. /// - One event. /// # + #[weight = SimpleDispatchInfo::default()] fn vest_other(origin, target: ::Source) -> DispatchResult { ensure_signed(origin)?; Self::update_lock(T::Lookup::lookup(target)?) @@ -450,7 +451,9 @@ mod tests { (12, 10, 20, 5 * self.existential_deposit) ], }.assimilate_storage(&mut t).unwrap(); - t.into() + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } } @@ -460,7 +463,6 @@ mod tests { .existential_deposit(256) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user1_free_balance = Balances::free_balance(&1); let user2_free_balance = Balances::free_balance(&2); let user12_free_balance = Balances::free_balance(&12); @@ -519,7 +521,6 @@ mod tests { .existential_deposit(10) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user1_free_balance = Balances::free_balance(&1); assert_eq!(user1_free_balance, 100); // Account 1 has free balance // Account 1 has only 5 units vested at block 1 (plus 50 unvested) @@ -537,7 +538,6 @@ mod tests { .existential_deposit(10) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user1_free_balance = Balances::free_balance(&1); assert_eq!(user1_free_balance, 100); // Account 1 has free balance // Account 1 has only 5 units vested at block 1 (plus 50 unvested) @@ -553,7 +553,6 @@ mod tests { .existential_deposit(10) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user1_free_balance = Balances::free_balance(&1); assert_eq!(user1_free_balance, 100); // Account 1 has free balance // Account 1 has only 5 units vested at block 1 (plus 50 unvested) @@ -569,7 +568,6 @@ mod tests { .existential_deposit(10) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); assert_ok!(Balances::transfer(Some(3).into(), 1, 100)); assert_ok!(Balances::transfer(Some(3).into(), 2, 100)); @@ -597,7 +595,6 @@ mod tests { .existential_deposit(256) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user12_free_balance = Balances::free_balance(&12); assert_eq!(user12_free_balance, 2560); // Account 12 has free balance @@ -623,7 +620,6 @@ mod tests { .existential_deposit(256) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user3_free_balance = Balances::free_balance(&3); let user4_free_balance = Balances::free_balance(&4); assert_eq!(user3_free_balance, 256 * 30); @@ -667,7 +663,6 @@ mod tests { .existential_deposit(256) .build() .execute_with(|| { - assert_eq!(System::block_number(), 1); let user2_free_balance = Balances::free_balance(&2); let user4_free_balance = Balances::free_balance(&4); assert_eq!(user2_free_balance, 256 * 20); diff --git a/primitives/allocator/Cargo.toml b/primitives/allocator/Cargo.toml index eee82771d66b0d5e156b8c0c4eff68772e18cf02..635a3c9128febcd6fd46690ea3e91043d935e70f 100644 --- a/primitives/allocator/Cargo.toml +++ b/primitives/allocator/Cargo.toml @@ -25,3 +25,6 @@ std = [ "log", "derive_more", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index 3e33295afe37d76d89314381be09d0685e13291d..fedeeceb3faadfb8e3ec6ff19b7204aa97e6904b 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate runtime api primitives" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-api-proc-macro = { version = "2.0.0-alpha.5", path = "proc-macro" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } @@ -32,3 +32,6 @@ std = [ "sp-version/std", "hash-db", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/proc-macro/Cargo.toml b/primitives/api/proc-macro/Cargo.toml index 841da7883297683aba1d88730303eddec7e8e538..25c5ae13435a5a107cee146acc2a0661afea8c9f 100644 --- a/primitives/api/proc-macro/Cargo.toml +++ b/primitives/api/proc-macro/Cargo.toml @@ -24,3 +24,6 @@ proc-macro-crate = "0.1.4" [features] default = [ "std" ] std = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index 459707bdb53f4425153d7dc1bc631566379fc092..e9f3087912e26f322214fa22cdbca397a4a2c332 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -19,7 +19,7 @@ use crate::utils::{ fold_fn_decl_for_client_side, extract_parameter_names_types_and_borrows, generate_native_call_generator_fn_name, return_type_extract_type, generate_method_runtime_api_impl_name, generate_call_api_at_fn_name, prefix_function_with_trait, - replace_wild_card_parameter_names, + replace_wild_card_parameter_names, AllowSelfRefInParameters, }; use proc_macro2::{TokenStream, Span}; @@ -198,7 +198,7 @@ fn generate_native_call_generators(decl: &ItemTrait) -> Result { // Generate a native call generator for each function of the given trait. for fn_ in fns { - let params = extract_parameter_names_types_and_borrows(&fn_)?; + let params = extract_parameter_names_types_and_borrows(&fn_, AllowSelfRefInParameters::No)?; let trait_fn_name = &fn_.ident; let fn_name = generate_native_call_generator_fn_name(&fn_.ident); let output = return_type_replace_block_with_node_block(fn_.output.clone()); @@ -592,7 +592,10 @@ impl<'a> ToClientSideDecl<'a> { // Get types and if the value is borrowed from all parameters. // If there is an error, we push it as the block to the user. - let param_types = match extract_parameter_names_types_and_borrows(fn_sig) { + let param_types = match extract_parameter_names_types_and_borrows( + fn_sig, + AllowSelfRefInParameters::No, + ) { Ok(res) => res.into_iter().map(|v| { let ty = v.1; let borrow = v.2; @@ -629,7 +632,10 @@ impl<'a> ToClientSideDecl<'a> { mut method: TraitItemMethod, context: TokenStream, ) -> TraitItemMethod { - let params = match extract_parameter_names_types_and_borrows(&method.sig) { + let params = match extract_parameter_names_types_and_borrows( + &method.sig, + AllowSelfRefInParameters::No, + ) { Ok(res) => res.into_iter().map(|v| v.0).collect::>(), Err(e) => { self.errors.push(e.to_compile_error()); @@ -780,7 +786,7 @@ fn generate_runtime_api_id(trait_name: &str) -> TokenStream { let mut res = [0; 8]; res.copy_from_slice(blake2_rfc::blake2b::blake2b(8, &[], trait_name.as_bytes()).as_bytes()); - quote!( const ID: [u8; 8] = [ #( #res ),* ]; ) + quote!( const ID: [u8; 8] = [ #( #res ),* ]; ) } /// Generates the const variable that holds the runtime api version. diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index e16cf3b5c46f78f50fbc03f5bff0f699dbedf503..7def6aa0fb73af3cca94b33ab1bb9ed7d6a8f70e 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -19,7 +19,8 @@ use crate::utils::{ generate_runtime_mod_name_for_trait, generate_method_runtime_api_impl_name, extract_parameter_names_types_and_borrows, generate_native_call_generator_fn_name, return_type_extract_type, generate_call_api_at_fn_name, prefix_function_with_trait, - extract_all_signature_types, + extract_all_signature_types, extract_block_type_from_trait_path, extract_impl_trait, + AllowSelfRefInParameters, RequireQualifiedTraitPath, }; use proc_macro2::{Span, TokenStream}; @@ -66,7 +67,7 @@ fn generate_impl_call( input: &Ident, impl_trait: &Path ) -> Result { - let params = extract_parameter_names_types_and_borrows(signature)?; + let params = extract_parameter_names_types_and_borrows(signature, AllowSelfRefInParameters::No)?; let c = generate_crate_access(HIDDEN_INCLUDES_ID); let c_iter = iter::repeat(&c); @@ -93,50 +94,6 @@ fn generate_impl_call( ) } -/// Extract the trait that is implemented in the given `ItemImpl`. -fn extract_impl_trait<'a>(impl_: &'a ItemImpl) -> Result<&'a Path> { - impl_.trait_.as_ref().map(|v| &v.1).ok_or_else( - || Error::new(impl_.span(), "Only implementation of traits are supported!") - ).and_then(|p| { - if p.segments.len() > 1 { - Ok(p) - } else { - Err( - Error::new( - p.span(), - "The implemented trait has to be referenced with a path, \ - e.g. `impl client::Core for Runtime`." - ) - ) - } - }) -} - -/// Extracts the runtime block identifier. -fn extract_runtime_block_ident(trait_: &Path) -> Result<&TypePath> { - let span = trait_.span(); - let generics = trait_ - .segments - .last() - .ok_or_else(|| Error::new(span, "Empty path not supported"))?; - - match &generics.arguments { - PathArguments::AngleBracketed(ref args) => { - args.args.first().and_then(|v| match v { - GenericArgument::Type(Type::Path(ref block)) => Some(block), - _ => None - }).ok_or_else(|| Error::new(args.span(), "Missing `Block` generic parameter.")) - }, - PathArguments::None => { - let span = trait_.segments.last().as_ref().unwrap().span(); - Err(Error::new(span, "Missing `Block` generic parameter.")) - }, - PathArguments::Parenthesized(_) => { - Err(Error::new(generics.arguments.span(), "Unexpected parentheses in path!")) - } - } -} - /// Generate all the implementation calls for the given functions. fn generate_impl_calls( impls: &[ItemImpl], @@ -145,7 +102,7 @@ fn generate_impl_calls( let mut impl_calls = Vec::new(); for impl_ in impls { - let impl_trait_path = extract_impl_trait(impl_)?; + let impl_trait_path = extract_impl_trait(impl_, RequireQualifiedTraitPath::Yes)?; let impl_trait = extend_with_runtime_decl_path(impl_trait_path.clone()); let impl_trait_ident = &impl_trait_path .segments @@ -307,11 +264,19 @@ fn generate_runtime_api_base_structures() -> Result { res } - fn runtime_version_at( + fn has_api( + &self, + at: &#crate_::BlockId, + ) -> std::result::Result where Self: Sized { + self.call.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, |v| v == A::VERSION)) + } + + fn has_api_with bool>( &self, at: &#crate_::BlockId, - ) -> std::result::Result<#crate_::RuntimeVersion, C::Error> { - self.call.runtime_version_at(at) + pred: P, + ) -> std::result::Result where Self: Sized { + self.call.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, pred)) } fn record_proof(&mut self) { @@ -450,7 +415,7 @@ fn generate_api_impl_for_runtime(impls: &[ItemImpl]) -> Result { // we put the `RuntimeBlock` as first argument for the trait generics. for impl_ in impls.iter() { let mut impl_ = impl_.clone(); - let trait_ = extract_impl_trait(&impl_)?.clone(); + let trait_ = extract_impl_trait(&impl_, RequireQualifiedTraitPath::Yes)?.clone(); let trait_ = extend_with_runtime_decl_path(trait_); impl_.trait_.as_mut().unwrap().1 = trait_; @@ -506,7 +471,10 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { }).collect::>() }; - let (param_types, error) = match extract_parameter_names_types_and_borrows(&input.sig) { + let (param_types, error) = match extract_parameter_names_types_and_borrows( + &input.sig, + AllowSelfRefInParameters::No, + ) { Ok(res) => ( res.into_iter().map(|v| { let ty = v.1; @@ -645,13 +613,13 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result let mut result = Vec::with_capacity(impls.len()); for impl_ in impls { - let impl_trait_path = extract_impl_trait(&impl_)?; + let impl_trait_path = extract_impl_trait(&impl_, RequireQualifiedTraitPath::Yes)?; let impl_trait = &impl_trait_path .segments .last() .ok_or_else(|| Error::new(impl_trait_path.span(), "Empty trait path not possible!"))? .clone(); - let runtime_block = extract_runtime_block_ident(impl_trait_path)?; + let runtime_block = extract_block_type_from_trait_path(impl_trait_path)?; let runtime_type = &impl_.self_ty; let mut runtime_mod_path = extend_with_runtime_decl_path(impl_trait_path.clone()); // remove the trait to get just the module path @@ -682,7 +650,9 @@ fn generate_runtime_api_versions(impls: &[ItemImpl]) -> Result { let mut processed_traits = HashSet::new(); for impl_ in impls { - let mut path = extend_with_runtime_decl_path(extract_impl_trait(&impl_)?.clone()); + let mut path = extend_with_runtime_decl_path( + extract_impl_trait(&impl_, RequireQualifiedTraitPath::Yes)?.clone(), + ); // Remove the trait let trait_ = path .segments diff --git a/primitives/api/proc-macro/src/lib.rs b/primitives/api/proc-macro/src/lib.rs index 9e5908717c18955cf45e3fefb4a95d12ea9d7fb8..12f435bd166dff2a579346ff8a710faec6bf89ba 100644 --- a/primitives/api/proc-macro/src/lib.rs +++ b/primitives/api/proc-macro/src/lib.rs @@ -21,6 +21,7 @@ use proc_macro::TokenStream; mod impl_runtime_apis; +mod mock_impl_runtime_apis; mod decl_runtime_apis; mod utils; @@ -29,6 +30,11 @@ pub fn impl_runtime_apis(input: TokenStream) -> TokenStream { impl_runtime_apis::impl_runtime_apis_impl(input) } +#[proc_macro] +pub fn mock_impl_runtime_apis(input: TokenStream) -> TokenStream { + mock_impl_runtime_apis::mock_impl_runtime_apis_impl(input) +} + #[proc_macro] pub fn decl_runtime_apis(input: TokenStream) -> TokenStream { decl_runtime_apis::decl_runtime_apis_impl(input) diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs new file mode 100644 index 0000000000000000000000000000000000000000..0767c804a637e2983c52affc97c69f4aea47ce62 --- /dev/null +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -0,0 +1,378 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::utils::{ + generate_crate_access, generate_hidden_includes, + generate_method_runtime_api_impl_name, extract_parameter_names_types_and_borrows, + return_type_extract_type, extract_block_type_from_trait_path, extract_impl_trait, + AllowSelfRefInParameters, RequireQualifiedTraitPath, +}; + +use proc_macro2::{Span, TokenStream}; + +use quote::quote; + +use syn::{ + spanned::Spanned, parse_macro_input, Ident, Type, ItemImpl, ImplItem, TypePath, parse_quote, + parse::{Parse, ParseStream, Result, Error}, fold::{self, Fold}, +}; + +/// Unique identifier used to make the hidden includes unique for this macro. +const HIDDEN_INCLUDES_ID: &str = "MOCK_IMPL_RUNTIME_APIS"; + +/// The structure used for parsing the runtime api implementations. +struct RuntimeApiImpls { + impls: Vec, +} + +impl Parse for RuntimeApiImpls { + fn parse(input: ParseStream) -> Result { + let mut impls = Vec::new(); + + while !input.is_empty() { + impls.push(ItemImpl::parse(input)?); + } + + if impls.is_empty() { + Err(Error::new(Span::call_site(), "No api implementation given!")) + } else { + Ok(Self { impls }) + } + } +} + +/// Implement the `ApiExt` trait, `ApiErrorExt` trait and the `Core` runtime api. +fn implement_common_api_traits( + error_type: Option, + block_type: TypePath, + self_ty: Type, +) -> Result { + let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID); + + let error_type = error_type.map(|e| quote!(#e)).unwrap_or_else(|| quote!(String)); + + Ok(quote!( + impl #crate_::ApiErrorExt for #self_ty { + type Error = #error_type; + } + + impl #crate_::ApiExt<#block_type> for #self_ty { + type StateBackend = #crate_::InMemoryBackend<#crate_::HashFor<#block_type>>; + + fn map_api_result std::result::Result, R, E>( + &self, + map_call: F, + ) -> std::result::Result where Self: Sized { + map_call(self) + } + + fn has_api( + &self, + _: &#crate_::BlockId<#block_type>, + ) -> std::result::Result where Self: Sized { + Ok(true) + } + + fn has_api_with bool>( + &self, + at: &#crate_::BlockId<#block_type>, + pred: P, + ) -> std::result::Result where Self: Sized { + Ok(pred(A::VERSION)) + } + + fn record_proof(&mut self) { + unimplemented!("`record_proof` not implemented for runtime api mocks") + } + + fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { + unimplemented!("`extract_proof` not implemented for runtime api mocks") + } + + fn into_storage_changes( + &self, + _: &Self::StateBackend, + _: Option<&#crate_::ChangesTrieState< + #crate_::HashFor<#block_type>, + #crate_::NumberFor<#block_type>, + >>, + _: <#block_type as #crate_::BlockT>::Hash, + ) -> std::result::Result< + #crate_::StorageChanges, + String + > where Self: Sized { + unimplemented!("`into_storage_changes` not implemented for runtime api mocks") + } + } + + impl #crate_::Core<#block_type> for #self_ty { + fn Core_version_runtime_api_impl( + &self, + _: &#crate_::BlockId<#block_type>, + _: #crate_::ExecutionContext, + _: Option<()>, + _: Vec, + ) -> std::result::Result<#crate_::NativeOrEncoded<#crate_::RuntimeVersion>, #error_type> { + unimplemented!("Not required for testing!") + } + + fn Core_execute_block_runtime_api_impl( + &self, + _: &#crate_::BlockId<#block_type>, + _: #crate_::ExecutionContext, + _: Option<#block_type>, + _: Vec, + ) -> std::result::Result<#crate_::NativeOrEncoded<()>, #error_type> { + unimplemented!("Not required for testing!") + } + + fn Core_initialize_block_runtime_api_impl( + &self, + _: &#crate_::BlockId<#block_type>, + _: #crate_::ExecutionContext, + _: Option<&<#block_type as #crate_::BlockT>::Header>, + _: Vec, + ) -> std::result::Result<#crate_::NativeOrEncoded<()>, #error_type> { + unimplemented!("Not required for testing!") + } + } + )) +} + +/// Auxialiry structure to fold a runtime api trait implementation into the expected format. +/// +/// This renames the methods, changes the method parameters and extracts the error type. +struct FoldRuntimeApiImpl<'a> { + /// The block type that is being used. + block_type: &'a TypePath, + /// The identifier of the trait being implemented. + impl_trait: &'a Ident, + /// Stores the error type that is being found in the trait implementation as associated type + /// with the name `Error`. + error_type: &'a mut Option, +} + +impl<'a> Fold for FoldRuntimeApiImpl<'a> { + fn fold_impl_item_method(&mut self, mut input: syn::ImplItemMethod) -> syn::ImplItemMethod { + let block = { + let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID); + + let (param_names, param_types, error) = match extract_parameter_names_types_and_borrows( + &input.sig, + AllowSelfRefInParameters::YesButIgnore, + ) { + Ok(res) => ( + res.iter().map(|v| v.0.clone()).collect::>(), + res.iter().map(|v| { + let ty = &v.1; + let borrow = &v.2; + quote!( #borrow #ty ) + }).collect::>(), + None + ), + Err(e) => (Vec::new(), Vec::new(), Some(e.to_compile_error())), + }; + + let block_type = &self.block_type; + + // Rewrite the input parameters. + input.sig.inputs = parse_quote! { + &self, + _: &#crate_::BlockId<#block_type>, + _: #crate_::ExecutionContext, + params: Option<( #( #param_types ),* )>, + _: Vec, + }; + + input.sig.ident = generate_method_runtime_api_impl_name( + &self.impl_trait, + &input.sig.ident, + ); + let ret_type = return_type_extract_type(&input.sig.output); + + // Generate the correct return type. + input.sig.output = parse_quote!( + -> std::result::Result<#crate_::NativeOrEncoded<#ret_type>, Self::Error> + ); + + let orig_block = input.block.clone(); + + // Generate the new method implementation that calls into the runtime. + parse_quote!( + { + // Get the error to the user (if we have one). + #error + + let (#( #param_names ),*) = params + .expect("Mocked runtime apis don't support calling deprecated api versions"); + + let __fn_implementation__ = move || #orig_block; + + Ok(#crate_::NativeOrEncoded::Native(__fn_implementation__())) + } + ) + }; + + let mut input = fold::fold_impl_item_method(self, input); + // We need to set the block, after we modified the rest of the ast, otherwise we would + // modify our generated block as well. + input.block = block; + input + } + + fn fold_impl_item(&mut self, input: ImplItem) -> ImplItem { + match input { + ImplItem::Type(ty) => { + if ty.ident == "Error" { + if let Some(error_type) = self.error_type { + if *error_type != ty.ty { + let error = Error::new( + ty.span(), + "Error type can not change between runtime apis", + ); + ImplItem::Verbatim(error.to_compile_error()) + } else { + ImplItem::Verbatim(Default::default()) + } + } else { + *self.error_type = Some(ty.ty); + ImplItem::Verbatim(Default::default()) + } + } else { + let error = Error::new( + ty.span(), + "Only associated type with name `Error` is allowed", + ); + ImplItem::Verbatim(error.to_compile_error()) + } + }, + o => fold::fold_impl_item(self, o), + } + } +} + +/// Result of [`generate_runtime_api_impls`]. +struct GeneratedRuntimeApiImpls { + /// All the runtime api implementations. + impls: TokenStream, + /// The error type that should be used by the runtime apis. + error_type: Option, + /// The block type that is being used by the runtime apis. + block_type: TypePath, + /// The type the traits are implemented for. + self_ty: Type, +} + +/// Generate the runtime api implementations from the given trait implementations. +/// +/// This folds the method names, changes the method parameters, method return type, +/// extracts the error type, self type and the block type. +fn generate_runtime_api_impls(impls: &[ItemImpl]) -> Result { + let mut result = Vec::with_capacity(impls.len()); + let mut error_type = None; + let mut global_block_type: Option = None; + let mut self_ty: Option> = None; + + for impl_ in impls { + let impl_trait_path = extract_impl_trait(&impl_, RequireQualifiedTraitPath::No)?; + let impl_trait = &impl_trait_path + .segments + .last() + .ok_or_else(|| Error::new(impl_trait_path.span(), "Empty trait path not possible!"))? + .clone(); + let block_type = extract_block_type_from_trait_path(impl_trait_path)?; + + self_ty = match self_ty.take() { + Some(self_ty) => { + if self_ty == impl_.self_ty { + Some(self_ty) + } else { + let mut error =Error::new( + impl_.self_ty.span(), + "Self type should not change between runtime apis", + ); + + error.combine(Error::new( + self_ty.span(), + "First self type found here", + )); + + return Err(error) + } + }, + None => Some(impl_.self_ty.clone()), + }; + + global_block_type = match global_block_type.take() { + Some(global_block_type) => { + if global_block_type == *block_type { + Some(global_block_type) + } else { + let mut error = Error::new( + block_type.span(), + "Block type should be the same between all runtime apis.", + ); + + error.combine(Error::new( + global_block_type.span(), + "First block type found here", + )); + + return Err(error) + } + }, + None => Some(block_type.clone()), + }; + + let mut visitor = FoldRuntimeApiImpl { + block_type, + impl_trait: &impl_trait.ident, + error_type: &mut error_type, + }; + + result.push(visitor.fold_item_impl(impl_.clone())); + } + + Ok(GeneratedRuntimeApiImpls { + impls: quote!( #( #result )* ), + error_type, + block_type: global_block_type.expect("There is a least one runtime api; qed"), + self_ty: *self_ty.expect("There is at least one runtime api; qed"), + }) +} + +/// The implementation of the `mock_impl_runtime_apis!` macro. +pub fn mock_impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + // Parse all impl blocks + let RuntimeApiImpls { impls: api_impls } = parse_macro_input!(input as RuntimeApiImpls); + + mock_impl_runtime_apis_impl_inner(&api_impls).unwrap_or_else(|e| e.to_compile_error()).into() +} + +fn mock_impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result { + let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID); + let GeneratedRuntimeApiImpls { impls, error_type, block_type, self_ty } = + generate_runtime_api_impls(api_impls)?; + let api_traits = implement_common_api_traits(error_type, block_type, self_ty)?; + + Ok(quote!( + #hidden_includes + + #impls + + #api_traits + )) +} diff --git a/primitives/api/proc-macro/src/utils.rs b/primitives/api/proc-macro/src/utils.rs index 8330624bf26d752679ce22afff35a989168a6379..1a79cf6c1ef5b5f046ac98229b3f4203f46ce45b 100644 --- a/primitives/api/proc-macro/src/utils.rs +++ b/primitives/api/proc-macro/src/utils.rs @@ -18,7 +18,7 @@ use proc_macro2::{TokenStream, Span}; use syn::{ Result, Ident, Signature, parse_quote, Type, Pat, spanned::Spanned, FnArg, Error, token::And, - ImplItem, ReturnType, + ImplItem, ReturnType, PathArguments, Path, GenericArgument, TypePath, ItemImpl, }; use quote::quote; @@ -126,13 +126,21 @@ pub fn generate_unique_pattern(pat: Pat, counter: &mut u32) -> Pat { }, _ => pat, } - } +} + +/// Allow `&self` in parameters of a method. +pub enum AllowSelfRefInParameters { + /// Allows `&self` in parameters, but doesn't return it as part of the parameters. + YesButIgnore, + No, +} /// Extracts the name, the type and `&` or ``(if it is a reference or not) /// for each parameter in the given function signature. -pub fn extract_parameter_names_types_and_borrows(sig: &Signature) - -> Result)>> -{ +pub fn extract_parameter_names_types_and_borrows( + sig: &Signature, + allow_self: AllowSelfRefInParameters, +) -> Result)>> { let mut result = Vec::new(); let mut generated_pattern_counter = 0; for input in sig.inputs.iter() { @@ -145,13 +153,20 @@ pub fn extract_parameter_names_types_and_borrows(sig: &Signature) t => { (t.clone(), None) }, }; - let name = - generate_unique_pattern((*arg.pat).clone(), &mut generated_pattern_counter); + let name = generate_unique_pattern( + (*arg.pat).clone(), + &mut generated_pattern_counter, + ); result.push((name, ty, borrow)); }, - FnArg::Receiver(_) => { + FnArg::Receiver(_) if matches!(allow_self, AllowSelfRefInParameters::No) => { return Err(Error::new(input.span(), "`self` parameter not supported!")) - } + }, + FnArg::Receiver(recv) => { + if recv.mutability.is_some() || recv.reference.is_none() { + return Err(Error::new(recv.span(), "Only `&self` is supported!")) + } + }, } } @@ -199,3 +214,60 @@ pub fn extract_all_signature_types(items: &[ImplItem]) -> Vec { .flatten() .collect() } + +/// Extracts the block type from a trait path. +/// +/// It is expected that the block type is the first type in the generic arguments. +pub fn extract_block_type_from_trait_path(trait_: &Path) -> Result<&TypePath> { + let span = trait_.span(); + let generics = trait_ + .segments + .last() + .ok_or_else(|| Error::new(span, "Empty path not supported"))?; + + match &generics.arguments { + PathArguments::AngleBracketed(ref args) => { + args.args.first().and_then(|v| match v { + GenericArgument::Type(Type::Path(ref block)) => Some(block), + _ => None + }).ok_or_else(|| Error::new(args.span(), "Missing `Block` generic parameter.")) + }, + PathArguments::None => { + let span = trait_.segments.last().as_ref().unwrap().span(); + Err(Error::new(span, "Missing `Block` generic parameter.")) + }, + PathArguments::Parenthesized(_) => { + Err(Error::new(generics.arguments.span(), "Unexpected parentheses in path!")) + }, + } +} + +/// Should a qualified trait path be required? +/// +/// e.g. `path::Trait` is qualified and `Trait` is not. +pub enum RequireQualifiedTraitPath { + Yes, + No, +} + +/// Extract the trait that is implemented by the given `ItemImpl`. +pub fn extract_impl_trait<'a>( + impl_: &'a ItemImpl, + require: RequireQualifiedTraitPath, +) -> Result<&'a Path> { + impl_.trait_.as_ref().map(|v| &v.1).ok_or_else( + || Error::new(impl_.span(), "Only implementation of traits are supported!") + ).and_then(|p| { + if p.segments.len() > 1 || matches!(require, RequireQualifiedTraitPath::No) { + Ok(p) + } else { + Err( + Error::new( + p.span(), + "The implemented trait has to be referenced with a path, \ + e.g. `impl client::Core for Runtime`." + ) + ) + } + }) +} diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 0901be5831dfd9758bdc59b466302963ad8f0fb8..74bcf19a9949e43316001d34b7ccfe3bda2b73cb 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -36,7 +36,7 @@ extern crate self as sp_api; #[doc(hidden)] #[cfg(feature = "std")] pub use sp_state_machine::{ - OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieState, + OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieState, InMemoryBackend, }; #[doc(hidden)] #[cfg(feature = "std")] @@ -78,7 +78,8 @@ use std::{panic::UnwindSafe, cell::RefCell}; /// declaration. Besides one exception, the macro adds an extra generic parameter `Block: BlockT` /// to the client side and the runtime side. This generic parameter is usable by the user. /// -/// For implementing these macros you should use the `impl_runtime_apis!` macro. +/// For implementing these macros you should use the +/// [`impl_runtime_apis!`](macro.impl_runtime_apis.html) macro. /// /// # Example /// @@ -143,8 +144,9 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// Tags given trait implementations as runtime apis. /// -/// All traits given to this macro, need to be declared with the `decl_runtime_apis!` macro. -/// The implementation of the trait should follow the declaration given to the `decl_runtime_apis!` +/// All traits given to this macro, need to be declared with the +/// [`decl_runtime_apis!`](macro.decl_runtime_apis.html) macro. The implementation of the trait +/// should follow the declaration given to the [`decl_runtime_apis!`](macro.decl_runtime_apis.html) /// macro, besides the `Block` type that is required as first generic parameter for each runtime /// api trait. When implementing a runtime api trait, it is required that the trait is referenced /// by a path, e.g. `impl my_trait::MyTrait for Runtime`. The macro will use this path to access @@ -182,7 +184,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// # } /// # pub trait BlockBuilder { /// # fn build_block() -> Block; -/// # } +/// # } /// # } /// /// /// All runtime api implementations need to be done in one call of the macro! @@ -226,6 +228,70 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// ``` pub use sp_api_proc_macro::impl_runtime_apis; +/// Mocks given trait implementations as runtime apis. +/// +/// Accepts similar syntax as [`impl_runtime_apis!`](macro.impl_runtime_apis.html) and generates +/// simplified mock implementations of the given runtime apis. The difference in syntax is that the +/// trait does not need to be referenced by a qualified path, methods accept the `&self` parameter +/// and the error type can be specified as associated type. If no error type is specified `String` +/// is used as error type. +/// +/// Besides implementing the given traits, the [`Core`], [`ApiExt`] and [`ApiErrorExt`] are +/// implemented automatically. +/// +/// # Example +/// +/// ```rust +/// use sp_version::create_runtime_str; +/// # +/// # use sp_runtime::traits::Block as BlockT; +/// # use sp_test_primitives::Block; +/// # +/// # sp_api::decl_runtime_apis! { +/// # /// Declare the api trait. +/// # pub trait Balance { +/// # /// Get the balance. +/// # fn get_balance() -> u64; +/// # /// Set the balance. +/// # fn set_balance(val: u64); +/// # } +/// # pub trait BlockBuilder { +/// # fn build_block() -> Block; +/// # } +/// # } +/// +/// struct MockApi { +/// balance: u64, +/// } +/// +/// /// All runtime api mock implementations need to be done in one call of the macro! +/// sp_api::mock_impl_runtime_apis! { +/// impl Balance for MockApi { +/// /// Here we take the `&self` to access the instance. +/// fn get_balance(&self) -> u64 { +/// self.balance +/// } +/// fn set_balance(_bal: u64) { +/// // Store the balance +/// } +/// } +/// +/// impl BlockBuilder for MockApi { +/// /// Sets the error type that is being used by the mock implementation. +/// /// The error type is used by all runtime apis. It is only required to +/// /// be specified in one trait implementation. +/// type Error = String; +/// +/// fn build_block() -> Block { +/// unimplemented!("Not Required in tests") +/// } +/// } +/// } +/// +/// # fn main() {} +/// ``` +pub use sp_api_proc_macro::mock_impl_runtime_apis; + /// A type that records all accessed trie nodes and generates a proof out of it. #[cfg(feature = "std")] pub type ProofRecorder = sp_state_machine::ProofRecorder>; @@ -293,21 +359,14 @@ pub trait ApiExt: ApiErrorExt { fn has_api( &self, at: &BlockId, - ) -> Result where Self: Sized { - self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, |v| v == A::VERSION)) - } + ) -> Result where Self: Sized; /// Check if the given api is implemented and the version passes a predicate. fn has_api_with bool>( &self, at: &BlockId, pred: P, - ) -> Result where Self: Sized { - self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, pred)) - } - - /// Returns the runtime version at the given block id. - fn runtime_version_at(&self, at: &BlockId) -> Result; + ) -> Result where Self: Sized; /// Start recording all accessed trie nodes for generating proofs. fn record_proof(&mut self); diff --git a/primitives/api/test/Cargo.toml b/primitives/api/test/Cargo.toml index 54d66470cf8fa72247e6414fa3913919cf2b4173..f2e66afc24abbac9f4c5604701d6b6f558bf3b4d 100644 --- a/primitives/api/test/Cargo.toml +++ b/primitives/api/test/Cargo.toml @@ -16,7 +16,7 @@ sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../blockchain" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } trybuild = "1.0.17" rustversion = "1.0.0" @@ -34,3 +34,6 @@ harness = false [features] default = [ "std" ] std = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/test/tests/decl_and_impl.rs b/primitives/api/test/tests/decl_and_impl.rs index d5a668dec1245a34ad054bfd7813298863aa07ca..a09bd0412c8afe9f15deb078a3f338dd0937ac59 100644 --- a/primitives/api/test/tests/decl_and_impl.rs +++ b/primitives/api/test/tests/decl_and_impl.rs @@ -14,7 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sp_api::{RuntimeApiInfo, decl_runtime_apis, impl_runtime_apis}; +use sp_api::{ + RuntimeApiInfo, decl_runtime_apis, impl_runtime_apis, mock_impl_runtime_apis, + ApiExt, +}; use sp_runtime::{traits::{GetNodeBlockType, Block as BlockT}, generic::BlockId}; @@ -81,6 +84,34 @@ impl_runtime_apis! { } } +struct MockApi { + block: Option, +} + +mock_impl_runtime_apis! { + impl Api for MockApi { + fn test(_: u64) { + unimplemented!() + } + + fn something_with_block(&self, _: Block) -> Block { + self.block.clone().unwrap() + } + + fn function_with_two_args(_: u64, _: Block) { + unimplemented!() + } + + fn same_name() {} + + fn wild_card(_: u32) {} + } + + impl ApiWithCustomVersion for MockApi { + fn same_name() {} + } +} + type TestClient = substrate_test_runtime_client::sc_client::Client< substrate_test_runtime_client::Backend, substrate_test_runtime_client::Executor, @@ -129,3 +160,22 @@ fn check_runtime_api_versions() { check_runtime_api_versions_contains::>(); check_runtime_api_versions_contains::>(); } + +#[test] +fn mock_runtime_api_has_api() { + let mock = MockApi { block: None }; + + assert!( + mock.has_api::>(&BlockId::Number(0)).unwrap(), + ); + assert!(mock.has_api::>(&BlockId::Number(0)).unwrap()); +} + +#[test] +#[should_panic(expected = "Mocked runtime apis don't support calling deprecated api versions")] +fn mock_runtime_api_panics_on_calling_old_version() { + let mock = MockApi { block: None }; + + #[allow(deprecated)] + let _ = mock.same_name_before_version_2(&BlockId::Number(0)); +} diff --git a/primitives/api/test/tests/ui/mock_only_error_associated_type.rs b/primitives/api/test/tests/ui/mock_only_error_associated_type.rs new file mode 100644 index 0000000000000000000000000000000000000000..bbd3c71c940177aba081954af7291c4f427bf604 --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_error_associated_type.rs @@ -0,0 +1,19 @@ +use substrate_test_runtime_client::runtime::Block; + +sp_api::decl_runtime_apis! { + pub trait Api { + fn test(data: u64); + } +} + +struct MockApi; + +sp_api::mock_impl_runtime_apis! { + impl Api for MockApi { + type OtherData = u32; + + fn test(data: u64) {} + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/mock_only_error_associated_type.stderr b/primitives/api/test/tests/ui/mock_only_error_associated_type.stderr new file mode 100644 index 0000000000000000000000000000000000000000..beced70413bb07cb873789ba48181d0d59d6ac32 --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_error_associated_type.stderr @@ -0,0 +1,5 @@ +error: Only associated type with name `Error` is allowed + --> $DIR/mock_only_error_associated_type.rs:13:3 + | +13 | type OtherData = u32; + | ^^^^ diff --git a/primitives/api/test/tests/ui/mock_only_one_block_type.rs b/primitives/api/test/tests/ui/mock_only_one_block_type.rs new file mode 100644 index 0000000000000000000000000000000000000000..969b21d7378d75b6b2bb28c1ca118aa041015784 --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_block_type.rs @@ -0,0 +1,27 @@ +use substrate_test_runtime_client::runtime::Block; + +struct Block2; + +sp_api::decl_runtime_apis! { + pub trait Api { + fn test(data: u64); + } + + pub trait Api2 { + fn test(data: u64); + } +} + +struct MockApi; + +sp_api::mock_impl_runtime_apis! { + impl Api for MockApi { + fn test(data: u64) {} + } + + impl Api2 for MockApi { + fn test(data: u64) {} + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/mock_only_one_block_type.stderr b/primitives/api/test/tests/ui/mock_only_one_block_type.stderr new file mode 100644 index 0000000000000000000000000000000000000000..1abc8db726a160e151cb47abf6b745d5630b1851 --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_block_type.stderr @@ -0,0 +1,19 @@ +error: Block type should be the same between all runtime apis. + --> $DIR/mock_only_one_block_type.rs:22:12 + | +22 | impl Api2 for MockApi { + | ^^^^^^ + +error: First block type found here + --> $DIR/mock_only_one_block_type.rs:18:11 + | +18 | impl Api for MockApi { + | ^^^^^ + +warning: unused import: `substrate_test_runtime_client::runtime::Block` + --> $DIR/mock_only_one_block_type.rs:1:5 + | +1 | use substrate_test_runtime_client::runtime::Block; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/mock_only_one_error_type.rs b/primitives/api/test/tests/ui/mock_only_one_error_type.rs new file mode 100644 index 0000000000000000000000000000000000000000..1c3f13dbb9bf1107e5c88fa50bd6ef12eb485b4c --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_error_type.rs @@ -0,0 +1,29 @@ +use substrate_test_runtime_client::runtime::Block; + +sp_api::decl_runtime_apis! { + pub trait Api { + fn test(data: u64); + } + + pub trait Api2 { + fn test(data: u64); + } +} + +struct MockApi; + +sp_api::mock_impl_runtime_apis! { + impl Api for MockApi { + type Error = u32; + + fn test(data: u64) {} + } + + impl Api2 for MockApi { + type Error = u64; + + fn test(data: u64) {} + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/mock_only_one_error_type.stderr b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr new file mode 100644 index 0000000000000000000000000000000000000000..8f026838c96b8db0e503179c745b3a76f4a2c181 --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr @@ -0,0 +1,27 @@ +error: Error type can not change between runtime apis + --> $DIR/mock_only_one_error_type.rs:23:3 + | +23 | type Error = u64; + | ^^^^ + +error[E0277]: the trait bound `u32: std::convert::From` is not satisfied + --> $DIR/mock_only_one_error_type.rs:15:1 + | +15 | / sp_api::mock_impl_runtime_apis! { +16 | | impl Api for MockApi { +17 | | type Error = u32; +18 | | +... | +26 | | } +27 | | } + | | ^ + | | | + | |_the trait `std::convert::From` is not implemented for `u32` + | in this macro invocation + | + = help: the following implementations were found: + > + > + > + > + and 16 others diff --git a/primitives/api/test/tests/ui/mock_only_one_self_type.rs b/primitives/api/test/tests/ui/mock_only_one_self_type.rs new file mode 100644 index 0000000000000000000000000000000000000000..4b29ec2a6ab07e7a08689549423e03687501543d --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_self_type.rs @@ -0,0 +1,26 @@ +use substrate_test_runtime_client::runtime::Block; + +sp_api::decl_runtime_apis! { + pub trait Api { + fn test(data: u64); + } + + pub trait Api2 { + fn test(data: u64); + } +} + +struct MockApi; +struct MockApi2; + +sp_api::mock_impl_runtime_apis! { + impl Api for MockApi { + fn test(data: u64) {} + } + + impl Api2 for MockApi2 { + fn test(data: u64) {} + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/mock_only_one_self_type.stderr b/primitives/api/test/tests/ui/mock_only_one_self_type.stderr new file mode 100644 index 0000000000000000000000000000000000000000..996d1d44c044c151bc5e57e93e83d3749e86a80c --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_one_self_type.stderr @@ -0,0 +1,19 @@ +error: Self type should not change between runtime apis + --> $DIR/mock_only_one_self_type.rs:21:23 + | +21 | impl Api2 for MockApi2 { + | ^^^^^^^^ + +error: First self type found here + --> $DIR/mock_only_one_self_type.rs:17:22 + | +17 | impl Api for MockApi { + | ^^^^^^^ + +warning: unused import: `substrate_test_runtime_client::runtime::Block` + --> $DIR/mock_only_one_self_type.rs:1:5 + | +1 | use substrate_test_runtime_client::runtime::Block; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.rs b/primitives/api/test/tests/ui/mock_only_self_reference.rs new file mode 100644 index 0000000000000000000000000000000000000000..8a733f5779ce938edcd1301b535938288b66737b --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_self_reference.rs @@ -0,0 +1,20 @@ +use substrate_test_runtime_client::runtime::Block; + +sp_api::decl_runtime_apis! { + pub trait Api { + fn test(data: u64); + fn test2(data: u64); + } +} + +struct MockApi; + +sp_api::mock_impl_runtime_apis! { + impl Api for MockApi { + fn test(self, data: u64) {} + + fn test2(&mut self, data: u64) {} + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr new file mode 100644 index 0000000000000000000000000000000000000000..9c1658b0a6cea1260b082eee7afea6d2521c66ac --- /dev/null +++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr @@ -0,0 +1,65 @@ +error: Only `&self` is supported! + --> $DIR/mock_only_self_reference.rs:14:11 + | +14 | fn test(self, data: u64) {} + | ^^^^ + +error: Only `&self` is supported! + --> $DIR/mock_only_self_reference.rs:16:12 + | +16 | fn test2(&mut self, data: u64) {} + | ^ + +error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for trait + --> $DIR/mock_only_self_reference.rs:12:1 + | +3 | / sp_api::decl_runtime_apis! { +4 | | pub trait Api { +5 | | fn test(data: u64); +6 | | fn test2(data: u64); +7 | | } +8 | | } + | |_- type in trait +... +12 | sp_api::mock_impl_runtime_apis! { + | -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | _expected `u64`, found `()` + | | +13 | | impl Api for MockApi { +14 | | fn test(self, data: u64) {} +15 | | +16 | | fn test2(&mut self, data: u64) {} +17 | | } +18 | | } + | |_- in this macro invocation + | + = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option, std::vec::Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>` + +error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for trait + --> $DIR/mock_only_self_reference.rs:12:1 + | +3 | / sp_api::decl_runtime_apis! { +4 | | pub trait Api { +5 | | fn test(data: u64); +6 | | fn test2(data: u64); +7 | | } +8 | | } + | |_- type in trait +... +12 | sp_api::mock_impl_runtime_apis! { + | -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | _expected `u64`, found `()` + | | +13 | | impl Api for MockApi { +14 | | fn test(self, data: u64) {} +15 | | +16 | | fn test2(&mut self, data: u64) {} +17 | | } +18 | | } + | |_- in this macro invocation + | + = note: expected fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option, std::vec::Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<()>, std::vec::Vec<_>) -> std::result::Result<_, _>` diff --git a/primitives/application-crypto/Cargo.toml b/primitives/application-crypto/Cargo.toml index c65584c2fbf95bcbbc6ffa50aead0c37f0cf7104..7c3fb5357997ffa2e7470c5aaf94cca168594fc8 100644 --- a/primitives/application-crypto/Cargo.toml +++ b/primitives/application-crypto/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/sp-application-crypto" [dependencies] sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } @@ -31,3 +31,6 @@ full_crypto = [ "sp-io/disable_panic_handler", "sp-io/disable_oom", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/application-crypto/src/ed25519.rs b/primitives/application-crypto/src/ed25519.rs index 414715a10684c18b49a33fe6bb1fc08c20639d4a..5be79ff4f7985a54e9eb0c34fb3552f3cf062bb1 100644 --- a/primitives/application-crypto/src/ed25519.rs +++ b/primitives/application-crypto/src/ed25519.rs @@ -23,12 +23,27 @@ use sp_std::vec::Vec; pub use sp_core::ed25519::*; mod app { + use sp_core::crypto::{CryptoTypePublicPair, Public as TraitPublic}; use sp_core::testing::ED25519; + use sp_core::ed25519::CRYPTO_ID; + crate::app_crypto!(super, ED25519); impl crate::traits::BoundToRuntimeAppPublic for Public { type Public = Self; } + + impl From for CryptoTypePublicPair { + fn from(key: Public) -> Self { + (&key).into() + } + } + + impl From<&Public> for CryptoTypePublicPair { + fn from(key: &Public) -> Self { + CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) + } + } } pub use app::{Public as AppPublic, Signature as AppSignature}; diff --git a/primitives/application-crypto/src/lib.rs b/primitives/application-crypto/src/lib.rs index b7c9ccaa9821aa49b9f14e70f841e05e1ef9853f..79572eb49d1e60d38a06b13356481c25a23803cf 100644 --- a/primitives/application-crypto/src/lib.rs +++ b/primitives/application-crypto/src/lib.rs @@ -21,11 +21,11 @@ #![cfg_attr(not(feature = "std"), no_std)] #[doc(hidden)] -pub use sp_core::{self, crypto::{CryptoType, Public, Derive, IsWrappedBy, Wraps}, RuntimeDebug}; +pub use sp_core::{self, crypto::{CryptoType, CryptoTypePublicPair, Public, Derive, IsWrappedBy, Wraps}, RuntimeDebug}; #[doc(hidden)] #[cfg(feature = "full_crypto")] pub use sp_core::crypto::{SecretStringError, DeriveJunction, Ss58Codec, Pair}; -pub use sp_core::{crypto::{KeyTypeId, key_types}}; +pub use sp_core::crypto::{CryptoTypeId, KeyTypeId, key_types}; #[doc(hidden)] pub use codec; @@ -103,17 +103,8 @@ macro_rules! app_crypto_pair { type Signature = Signature; type DeriveError = <$pair as $crate::Pair>::DeriveError; - #[cfg(feature = "std")] - fn generate_with_phrase(password: Option<&str>) -> (Self, String, Self::Seed) { - let r = <$pair>::generate_with_phrase(password); - (Self(r.0), r.1, r.2) - } - #[cfg(feature = "std")] - fn from_phrase(phrase: &str, password: Option<&str>) - -> Result<(Self, Self::Seed), $crate::SecretStringError> - { - <$pair>::from_phrase(phrase, password).map(|r| (Self(r.0), r.1)) - } + $crate::app_crypto_pair_functions_if_std!($pair); + fn derive< Iter: Iterator >(&self, path: Iter, seed: Option) -> Result<(Self, Option), Self::DeriveError> { @@ -158,10 +149,38 @@ macro_rules! app_crypto_pair { }; } +/// Implements functions for the `Pair` trait when `feature = "std"` is enabled. +#[doc(hidden)] +#[cfg(feature = "std")] +#[macro_export] +macro_rules! app_crypto_pair_functions_if_std { + ($pair:ty) => { + fn generate_with_phrase(password: Option<&str>) -> (Self, String, Self::Seed) { + let r = <$pair>::generate_with_phrase(password); + (Self(r.0), r.1, r.2) + } + + fn from_phrase(phrase: &str, password: Option<&str>) + -> Result<(Self, Self::Seed), $crate::SecretStringError> + { + <$pair>::from_phrase(phrase, password).map(|r| (Self(r.0), r.1)) + } + } +} + +#[doc(hidden)] +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! app_crypto_pair_functions_if_std { + ($pair:ty) => {} +} + + /// Declares Public type which is functionally equivalent to `$public`, but is new /// Application-specific type whose identifier is `$key_type`. /// can only be used together with `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_full_crypto { ($public:ty, $key_type:expr) => { @@ -195,6 +214,7 @@ macro_rules! app_crypto_public_full_crypto { /// Application-specific type whose identifier is `$key_type`. /// can only be used without `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_not_full_crypto { ($public:ty, $key_type:expr) => { @@ -223,44 +243,11 @@ macro_rules! app_crypto_public_not_full_crypto { /// Declares Public type which is functionally equivalent to `$public`, but is new /// Application-specific type whose identifier is `$key_type`. /// For full functionality, app_crypto_public_(not)_full_crypto! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_common { ($public:ty, $sig:ty, $key_type:expr) => { - impl $crate::Derive for Public { - #[cfg(feature = "std")] - fn derive>(&self, - path: Iter - ) -> Option { - self.0.derive(path).map(Self) - } - } - - #[cfg(feature = "std")] - impl std::fmt::Display for Public { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - use $crate::Ss58Codec; - write!(f, "{}", self.0.to_ss58check()) - } - } - #[cfg(feature = "std")] - impl $crate::serde::Serialize for Public { - fn serialize(&self, serializer: S) -> std::result::Result where - S: $crate::serde::Serializer - { - use $crate::Ss58Codec; - serializer.serialize_str(&self.to_ss58check()) - } - } - #[cfg(feature = "std")] - impl<'de> $crate::serde::Deserialize<'de> for Public { - fn deserialize(deserializer: D) -> std::result::Result where - D: $crate::serde::Deserializer<'de> - { - use $crate::Ss58Codec; - Public::from_ss58check(&String::deserialize(deserializer)?) - .map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e))) - } - } + $crate::app_crypto_public_common_if_std!(); impl AsRef<[u8]> for Public { fn as_ref(&self) -> &[u8] { self.0.as_ref() } @@ -309,10 +296,63 @@ macro_rules! app_crypto_public_common { } } +/// Implements traits for the public key type if `feature = "std"` is enabled. +#[cfg(feature = "std")] +#[doc(hidden)] +#[macro_export] +macro_rules! app_crypto_public_common_if_std { + () => { + impl $crate::Derive for Public { + fn derive>(&self, + path: Iter + ) -> Option { + self.0.derive(path).map(Self) + } + } + + impl std::fmt::Display for Public { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use $crate::Ss58Codec; + write!(f, "{}", self.0.to_ss58check()) + } + } + + impl $crate::serde::Serialize for Public { + fn serialize(&self, serializer: S) -> std::result::Result where + S: $crate::serde::Serializer + { + use $crate::Ss58Codec; + serializer.serialize_str(&self.to_ss58check()) + } + } + + impl<'de> $crate::serde::Deserialize<'de> for Public { + fn deserialize(deserializer: D) -> std::result::Result where + D: $crate::serde::Deserializer<'de> + { + use $crate::Ss58Codec; + Public::from_ss58check(&String::deserialize(deserializer)?) + .map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e))) + } + } + } +} + +#[cfg(not(feature = "std"))] +#[doc(hidden)] +#[macro_export] +macro_rules! app_crypto_public_common_if_std { + () => { + impl $crate::Derive for Public {} + } +} + + /// Declares Signature type which is functionally equivalent to `$sig`, but is new /// Application-specific type whose identifier is `$key_type`. /// can only be used together with `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_full_crypto { ($sig:ty, $key_type:expr) => { @@ -345,6 +385,7 @@ macro_rules! app_crypto_signature_full_crypto { /// Application-specific type whose identifier is `$key_type`. /// can only be used without `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_not_full_crypto { ($sig:ty, $key_type:expr) => { @@ -372,6 +413,7 @@ macro_rules! app_crypto_signature_not_full_crypto { /// Declares Signature type which is functionally equivalent to `$sig`, but is new /// Application-specific type whose identifier is `$key_type`. /// For full functionality, app_crypto_public_(not)_full_crypto! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_common { ($sig:ty, $key_type:expr) => { diff --git a/primitives/application-crypto/src/sr25519.rs b/primitives/application-crypto/src/sr25519.rs index 59c6f19b6f2668939054127347f5a87ed6989907..a0f2cef1c4e45b6150be335ac271e3dd592febbe 100644 --- a/primitives/application-crypto/src/sr25519.rs +++ b/primitives/application-crypto/src/sr25519.rs @@ -23,12 +23,27 @@ use sp_std::vec::Vec; pub use sp_core::sr25519::*; mod app { + use sp_core::crypto::{CryptoTypePublicPair, Public as TraitPublic}; use sp_core::testing::SR25519; + use sp_core::sr25519::CRYPTO_ID; + crate::app_crypto!(super, SR25519); impl crate::traits::BoundToRuntimeAppPublic for Public { type Public = Self; } + + impl From for CryptoTypePublicPair { + fn from(key: Public) -> Self { + (&key).into() + } + } + + impl From<&Public> for CryptoTypePublicPair { + fn from(key: &Public) -> Self { + CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) + } + } } pub use app::{Public as AppPublic, Signature as AppSignature}; diff --git a/primitives/application-crypto/test/Cargo.toml b/primitives/application-crypto/test/Cargo.toml index fae91e678355735d6e42a2525e9fe9a1cd4fda49..d34840b4eb5206429fbf829f60c2e11187b838f4 100644 --- a/primitives/application-crypto/test/Cargo.toml +++ b/primitives/application-crypto/test/Cargo.toml @@ -15,3 +15,6 @@ substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-u sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } sp-api = { version = "2.0.0-alpha.5", path = "../../api" } sp-application-crypto = { version = "2.0.0-alpha.5", path = "../" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/application-crypto/test/src/ed25519.rs b/primitives/application-crypto/test/src/ed25519.rs index 400edfd6ae4402f9ab77fc0fe4fde9f61dfd89ce..1d72962829a53309ed4146a5f8657820ba15b697 100644 --- a/primitives/application-crypto/test/src/ed25519.rs +++ b/primitives/application-crypto/test/src/ed25519.rs @@ -17,7 +17,10 @@ //! Integration tests for ed25519 use sp_runtime::generic::BlockId; -use sp_core::{testing::{KeyStore, ED25519}, crypto::Pair}; +use sp_core::{ + crypto::Pair, + testing::{KeyStore, ED25519}, +}; use substrate_test_runtime_client::{ TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt, runtime::TestAPI, @@ -33,8 +36,7 @@ fn ed25519_works_in_runtime() { .test_ed25519_crypto(&BlockId::Number(0)) .expect("Tests `ed25519` crypto."); - let key_pair = keystore.read().ed25519_key_pair(ED25519, &public.as_ref()) - .expect("There should be at a `ed25519` key in the keystore for the given public key."); - - assert!(AppPair::verify(&signature, "ed25519", &AppPublic::from(key_pair.public()))); + let supported_keys = keystore.read().keys(ED25519).unwrap(); + assert!(supported_keys.contains(&public.clone().into())); + assert!(AppPair::verify(&signature, "ed25519", &AppPublic::from(public))); } diff --git a/primitives/application-crypto/test/src/sr25519.rs b/primitives/application-crypto/test/src/sr25519.rs index 49bb3c2a8364027c1dd4eceef607deb49ca09469..f2c7c48b2bc91c0329ecaad7541be5f13f3f754c 100644 --- a/primitives/application-crypto/test/src/sr25519.rs +++ b/primitives/application-crypto/test/src/sr25519.rs @@ -18,7 +18,10 @@ use sp_runtime::generic::BlockId; -use sp_core::{testing::{KeyStore, SR25519}, crypto::Pair}; +use sp_core::{ + crypto::Pair, + testing::{KeyStore, SR25519}, +}; use substrate_test_runtime_client::{ TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt, runtime::TestAPI, @@ -34,8 +37,7 @@ fn sr25519_works_in_runtime() { .test_sr25519_crypto(&BlockId::Number(0)) .expect("Tests `sr25519` crypto."); - let key_pair = keystore.read().sr25519_key_pair(SR25519, public.as_ref()) - .expect("There should be at a `sr25519` key in the keystore for the given public key."); - - assert!(AppPair::verify(&signature, "sr25519", &AppPublic::from(key_pair.public()))); + let supported_keys = keystore.read().keys(SR25519).unwrap(); + assert!(supported_keys.contains(&public.clone().into())); + assert!(AppPair::verify(&signature, "sr25519", &AppPublic::from(public))); } diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index f41cac4fb5ce6f312b87e9218b2dbfb933cab9a3..208525f6c193bdca2b9b5f9ba4a566999d523544 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/sp-arithmetic" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } integer-sqrt = "0.1.2" num-traits = { version = "0.2.8", default-features = false } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } @@ -36,3 +36,6 @@ std = [ [[bench]] name = "bench" harness = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/arithmetic/fuzzer/Cargo.lock b/primitives/arithmetic/fuzzer/Cargo.lock index b196418375c59ee17bb8ba03680011d5b0efd1da..3a4187437ae7312ec7016cb9e41638a018cfcbbf 100644 --- a/primitives/arithmetic/fuzzer/Cargo.lock +++ b/primitives/arithmetic/fuzzer/Cargo.lock @@ -32,9 +32,9 @@ checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" [[package]] name = "byteorder" -version = "1.3.2" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "c2-chacha" @@ -83,8 +83,6 @@ dependencies = [ [[package]] name = "honggfuzz" version = "0.5.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c27b4aa3049d6d10d8e33d52c9d03ca9aec18f8a449b246f8c4a5b0c10fb34" dependencies = [ "arbitrary", "lazy_static", @@ -114,9 +112,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.66" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" [[package]] name = "memmap" @@ -160,9 +158,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "1.1.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f747c06d9f3b2ad387ac881b9667298c81b1243aa9833f086e05996937c35507" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" dependencies = [ "arrayvec", "bitvec", @@ -173,9 +171,9 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "1.1.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e513ff3e406f3ede6796dcdc83d0b32ffb86668cea1ccf7363118abeb00476" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -211,18 +209,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" dependencies = [ "unicode-xid", ] [[package]] name = "quote" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" dependencies = [ "proc-macro2", ] @@ -296,7 +294,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "integer-sqrt", "num-traits", @@ -319,7 +317,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "proc-macro2", "quote", @@ -328,7 +326,7 @@ dependencies = [ [[package]] name = "sp-std" -version = "2.0.0" +version = "2.0.0-alpha.3" [[package]] name = "static_assertions" @@ -338,9 +336,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" dependencies = [ "proc-macro2", "quote", diff --git a/primitives/arithmetic/fuzzer/Cargo.toml b/primitives/arithmetic/fuzzer/Cargo.toml index ea129906ea279f241c63454f453a6d496960b0f6..26b5f8c27b5085fd615dce5af70939e994c76767 100644 --- a/primitives/arithmetic/fuzzer/Cargo.toml +++ b/primitives/arithmetic/fuzzer/Cargo.toml @@ -1,23 +1,32 @@ [package] name = "sp-arithmetic-fuzzer" -version = "2.0.0" +version = "2.0.0-alpha.5" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Fuzzer for fixed point arithmetic primitives." +documentation = "https://docs.rs/sp-arithmetic-fuzzer" [dependencies] -sp-arithmetic = { version = "2.0.0", path = ".." } +sp-arithmetic = { version = "2.0.0-alpha.5", path = ".." } honggfuzz = "0.5" primitive-types = "0.7.0" num-bigint = "0.2" num-traits = "0.2" -[workspace] - [[bin]] name = "biguint" path = "src/biguint.rs" +[[bin]] +name = "per_thing_rational" +path = "src/per_thing_rational.rs" + [[bin]] name = "rational128" path = "src/rational128.rs" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/arithmetic/fuzzer/src/per_thing_rational.rs b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs new file mode 100644 index 0000000000000000000000000000000000000000..c2dda3de2299cb713114a856c6eafbedf2d91be2 --- /dev/null +++ b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs @@ -0,0 +1,122 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Running +//! Running this fuzzer can be done with `cargo hfuzz run per_thing_rational`. `honggfuzz` CLI options can +//! be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads. +//! +//! # Debugging a panic +//! Once a panic is found, it can be debugged with +//! `cargo hfuzz run-debug per_thing_rational hfuzz_workspace/per_thing_rational/*.fuzz`. + +use honggfuzz::fuzz; +use sp_arithmetic::{ + PerThing, PerU16, Percent, Perbill, Perquintill, traits::SaturatedConversion, +}; + +fn main() { + loop { + fuzz!(| + data: ((u16, u16), (u32, u32), (u64, u64)) + | { + + let (u16_pair, u32_pair, u64_pair) = data; + + // peru16 + let (smaller, bigger) = (u16_pair.0.min(u16_pair.1), u16_pair.0.max(u16_pair.1)); + let ratio = PerU16::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + PerU16::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + let (smaller, bigger) = (u32_pair.0.min(u32_pair.1), u32_pair.0.max(u32_pair.1)); + let ratio = PerU16::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + PerU16::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + let (smaller, bigger) = (u64_pair.0.min(u64_pair.1), u64_pair.0.max(u64_pair.1)); + let ratio = PerU16::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + PerU16::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + + // percent + let (smaller, bigger) = (u16_pair.0.min(u16_pair.1), u16_pair.0.max(u16_pair.1)); + let ratio = Percent::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Percent::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + + let (smaller, bigger) = (u32_pair.0.min(u32_pair.1), u32_pair.0.max(u32_pair.1)); + let ratio = Percent::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Percent::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + + let (smaller, bigger) = (u64_pair.0.min(u64_pair.1), u64_pair.0.max(u64_pair.1)); + let ratio = Percent::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Percent::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1, + ); + + // perbill + let (smaller, bigger) = (u32_pair.0.min(u32_pair.1), u32_pair.0.max(u32_pair.1)); + let ratio = Perbill::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Perbill::from_fraction(smaller as f64 / bigger.max(1) as f64), + 100, + ); + + let (smaller, bigger) = (u64_pair.0.min(u64_pair.1), u64_pair.0.max(u64_pair.1)); + let ratio = Perbill::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Perbill::from_fraction(smaller as f64 / bigger.max(1) as f64), + 100, + ); + + // perquintillion + let (smaller, bigger) = (u64_pair.0.min(u64_pair.1), u64_pair.0.max(u64_pair.1)); + let ratio = Perquintill::from_rational_approximation(smaller, bigger); + assert_per_thing_equal_error( + ratio, + Perquintill::from_fraction(smaller as f64 / bigger.max(1) as f64), + 1000, + ); + + }) + } +} + +fn assert_per_thing_equal_error(a: T, b: T, err: u128) { + let a_abs = a.deconstruct().saturated_into::(); + let b_abs = b.deconstruct().saturated_into::(); + let diff = a_abs.max(b_abs) - a_abs.min(b_abs); + dbg!(&diff); + assert!(diff <= err, "{:?} !~ {:?}", a, b); +} diff --git a/primitives/arithmetic/src/fixed64.rs b/primitives/arithmetic/src/fixed64.rs index eea1ab68a3bb2272714fb0e3e506201634ffcaa0..6b399b6aa5106d3889c1152c8cb1e7fc38a14585 100644 --- a/primitives/arithmetic/src/fixed64.rs +++ b/primitives/arithmetic/src/fixed64.rs @@ -110,12 +110,18 @@ impl Saturating for Fixed64 { fn saturating_add(self, rhs: Self) -> Self { Self(self.0.saturating_add(rhs.0)) } + fn saturating_mul(self, rhs: Self) -> Self { Self(self.0.saturating_mul(rhs.0) / DIV) } + fn saturating_sub(self, rhs: Self) -> Self { Self(self.0.saturating_sub(rhs.0)) } + + fn saturating_pow(self, exp: usize) -> Self { + Self(self.0.saturating_pow(exp as u32)) + } } /// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index c2feae00b74ff9a3e0d016558c2dfbaee2e1dffd..f6d8b53e3499b340b27285964e8c366be39ca158 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -19,7 +19,7 @@ #![cfg_attr(not(feature = "std"), no_std)] /// Copied from `sp-runtime` and documented there. -#[cfg(test)] +#[macro_export] macro_rules! assert_eq_error_rate { ($x:expr, $y:expr, $error:expr $(,)?) => { assert!( @@ -40,5 +40,17 @@ mod fixed64; mod rational128; pub use fixed64::Fixed64; -pub use per_things::{PerThing, Percent, Permill, Perbill, Perquintill}; +pub use per_things::{PerThing, Percent, PerU16, Permill, Perbill, Perquintill}; pub use rational128::Rational128; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn peru16_rational_does_not_overflow() { + // A historical example that will panic only for per_thing type that are created with + // maximum capacity of their type, e.g. PerU16. + let _ = PerU16::from_rational_approximation(17424870u32, 17424870); + } +} diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index cbb804baf5d49e97ce7d9817b7169dffe6e87643..ad529fbf32e249ef1fb0e5838c30fce52aab4b25 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -17,42 +17,160 @@ #[cfg(feature = "std")] use serde::{Serialize, Deserialize}; -use sp_std::{ops, prelude::*, convert::TryInto}; -use codec::{Encode, Decode, CompactAs}; +use sp_std::{ops, fmt, prelude::*, convert::TryInto}; +use codec::{Encode, CompactAs}; use crate::traits::{ - SaturatedConversion, UniqueSaturatedInto, Saturating, BaseArithmetic, + SaturatedConversion, UniqueSaturatedInto, Saturating, BaseArithmetic, Bounded, Zero, }; use sp_debug_derive::RuntimeDebug; /// Something that implements a fixed point ration with an arbitrary granularity `X`, as _parts per /// `X`_. -pub trait PerThing: Sized + Saturating + Copy { +pub trait PerThing: + Sized + Saturating + Copy + Default + Eq + PartialEq + Ord + PartialOrd + Bounded + fmt::Debug +{ /// The data type used to build this per-thingy. - type Inner: BaseArithmetic + Copy; + type Inner: BaseArithmetic + Copy + fmt::Debug; - /// accuracy of this type - const ACCURACY: Self::Inner; + /// A data type larger than `Self::Inner`, used to avoid overflow in some computations. + /// It must be able to compute `ACCURACY^2`. + type Upper: BaseArithmetic + Copy + From + TryInto + fmt::Debug; - /// NoThing - fn zero() -> Self; + /// The accuracy of this type. + const ACCURACY: Self::Inner; - /// `true` if this is nothing. - fn is_zero(&self) -> bool; + /// Equivalent to `Self::from_parts(0)`. + fn zero() -> Self { Self::from_parts(Self::Inner::zero()) } - /// Everything. - fn one() -> Self; + /// Return `true` if this is nothing. + fn is_zero(&self) -> bool { self.deconstruct() == Self::Inner::zero() } - /// Consume self and deconstruct into a raw numeric type. - fn deconstruct(self) -> Self::Inner; + /// Equivalent to `Self::from_parts(Self::ACCURACY)`. + fn one() -> Self { Self::from_parts(Self::ACCURACY) } - /// From an explicitly defined number of parts per maximum of the type. - fn from_parts(parts: Self::Inner) -> Self; + /// Return `true` if this is one. + fn is_one(&self) -> bool { self.deconstruct() == Self::ACCURACY } - /// Converts a percent into `Self`. Equal to `x / 100`. - fn from_percent(x: Self::Inner) -> Self; + /// Build this type from a percent. Equivalent to `Self::from_parts(x * Self::ACCURACY / 100)` + /// but more accurate. + fn from_percent(x: Self::Inner) -> Self { + let a = x.min(100.into()); + let b = Self::ACCURACY; + // if Self::ACCURACY % 100 > 0 then we need the correction for accuracy + let c = rational_mul_correction::(b, a, 100.into(), Rounding::Nearest); + Self::from_parts(a / 100.into() * b + c) + } /// Return the product of multiplication of this value by itself. - fn square(self) -> Self; + fn square(self) -> Self { + let p = Self::Upper::from(self.deconstruct()); + let q = Self::Upper::from(Self::ACCURACY); + Self::from_rational_approximation(p * p, q * q) + } + + /// Multiplication that always rounds down to a whole number. The standard `Mul` rounds to the + /// nearest whole number. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// // round to nearest + /// assert_eq!(Percent::from_percent(34) * 10u64, 3); + /// assert_eq!(Percent::from_percent(36) * 10u64, 4); + /// + /// // round down + /// assert_eq!(Percent::from_percent(34).mul_floor(10u64), 3); + /// assert_eq!(Percent::from_percent(36).mul_floor(10u64), 3); + /// # } + /// ``` + fn mul_floor(self, b: N) -> N + where N: Clone + From + UniqueSaturatedInto + ops::Rem + + ops::Div + ops::Mul + ops::Add + { + overflow_prune_mul::(b, self.deconstruct(), Rounding::Down) + } + + /// Multiplication that always rounds the result up to a whole number. The standard `Mul` + /// rounds to the nearest whole number. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// // round to nearest + /// assert_eq!(Percent::from_percent(34) * 10u64, 3); + /// assert_eq!(Percent::from_percent(36) * 10u64, 4); + /// + /// // round up + /// assert_eq!(Percent::from_percent(34).mul_ceil(10u64), 4); + /// assert_eq!(Percent::from_percent(36).mul_ceil(10u64), 4); + /// # } + /// ``` + fn mul_ceil(self, b: N) -> N + where N: Clone + From + UniqueSaturatedInto + ops::Rem + + ops::Div + ops::Mul + ops::Add + { + overflow_prune_mul::(b, self.deconstruct(), Rounding::Up) + } + + /// Saturating multiplication by the reciprocal of `self`. The result is rounded to the + /// nearest whole number and saturates at the numeric bounds instead of overflowing. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// assert_eq!(Percent::from_percent(50).saturating_reciprocal_mul(10u64), 20); + /// # } + /// ``` + fn saturating_reciprocal_mul(self, b: N) -> N + where N: Clone + From + UniqueSaturatedInto + ops::Rem + + ops::Div + ops::Mul + ops::Add + Saturating + { + saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Nearest) + } + + /// Saturating multiplication by the reciprocal of `self`. The result is rounded down to the + /// nearest whole number and saturates at the numeric bounds instead of overflowing. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// // round to nearest + /// assert_eq!(Percent::from_percent(60).saturating_reciprocal_mul(10u64), 17); + /// // round down + /// assert_eq!(Percent::from_percent(60).saturating_reciprocal_mul_floor(10u64), 16); + /// # } + /// ``` + fn saturating_reciprocal_mul_floor(self, b: N) -> N + where N: Clone + From + UniqueSaturatedInto + ops::Rem + + ops::Div + ops::Mul + ops::Add + Saturating + { + saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Down) + } + + /// Saturating multiplication by the reciprocal of `self`. The result is rounded up to the + /// nearest whole number and saturates at the numeric bounds instead of overflowing. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// // round to nearest + /// assert_eq!(Percent::from_percent(61).saturating_reciprocal_mul(10u64), 16); + /// // round up + /// assert_eq!(Percent::from_percent(61).saturating_reciprocal_mul_ceil(10u64), 17); + /// # } + /// ``` + fn saturating_reciprocal_mul_ceil(self, b: N) -> N + where N: Clone + From + UniqueSaturatedInto + ops::Rem + + ops::Div + ops::Mul + ops::Add + Saturating + { + saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Up) + } + + /// Consume self and return the number of parts per thing. + fn deconstruct(self) -> Self::Inner; + + /// Build this type from a number of parts per thing. + fn from_parts(parts: Self::Inner) -> Self; /// Converts a fraction into `Self`. #[cfg(feature = "std")] @@ -63,88 +181,190 @@ pub trait PerThing: Sized + Saturating + Copy { /// The computation of this approximation is performed in the generic type `N`. Given /// `M` as the data type that can hold the maximum value of this per-thing (e.g. u32 for /// perbill), this can only work if `N == M` or `N: From + TryInto`. + /// + /// Note that this always rounds _down_, i.e. + /// + /// ```rust + /// # use sp_arithmetic::{Percent, PerThing}; + /// # fn main () { + /// // 989/100 is technically closer to 99%. + /// assert_eq!( + /// Percent::from_rational_approximation(989, 1000), + /// Percent::from_parts(98), + /// ); + /// # } + /// ``` fn from_rational_approximation(p: N, q: N) -> Self - where N: Clone + Ord + From + TryInto + ops::Div; + where N: Clone + Ord + From + TryInto + TryInto + + ops::Div + ops::Rem + ops::Add; +} + +/// The rounding method to use. +/// +/// `PerThing`s are unsigned so `Up` means towards infinity and `Down` means towards zero. +/// `Nearest` will round an exact half down. +enum Rounding { + Up, + Down, + Nearest, +} + +/// Saturating reciprocal multiplication. Compute `x / self`, saturating at the numeric +/// bounds instead of overflowing. +fn saturating_reciprocal_mul( + x: N, + part: P::Inner, + rounding: Rounding, +) -> N +where + N: Clone + From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem + Saturating, + P: PerThing, +{ + let maximum: N = P::ACCURACY.into(); + let c = rational_mul_correction::( + x.clone(), + P::ACCURACY, + part, + rounding, + ); + (x / part.into()).saturating_mul(maximum).saturating_add(c) +} + +/// Overflow-prune multiplication. Accurately multiply a value by `self` without overflowing. +fn overflow_prune_mul( + x: N, + part: P::Inner, + rounding: Rounding, +) -> N +where + N: Clone + From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem, + P: PerThing, +{ + let maximum: N = P::ACCURACY.into(); + let part_n: N = part.into(); + let c = rational_mul_correction::( + x.clone(), + part, + P::ACCURACY, + rounding, + ); + (x / maximum) * part_n + c +} + +/// Compute the error due to integer division in the expression `x / denom * numer`. +/// +/// Take the remainder of `x / denom` and multiply by `numer / denom`. The result can be added +/// to `x / denom * numer` for an accurate result. +fn rational_mul_correction( + x: N, + numer: P::Inner, + denom: P::Inner, + rounding: Rounding, +) -> N +where + N: From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem, + P: PerThing, +{ + let numer_upper = P::Upper::from(numer); + let denom_n = N::from(denom); + let denom_upper = P::Upper::from(denom); + let rem = x.rem(denom_n); + // `rem` is less than `denom`, which fits in `P::Inner`. + let rem_inner = rem.saturated_into::(); + // `P::Upper` always fits `P::Inner::max_value().pow(2)`, thus it fits `rem * numer`. + let rem_mul_upper = P::Upper::from(rem_inner) * numer_upper; + // `rem` is less than `denom`, so `rem * numer / denom` is less than `numer`, which fits in + // `P::Inner`. + let mut rem_mul_div_inner = (rem_mul_upper / denom_upper).saturated_into::(); + match rounding { + // Already rounded down + Rounding::Down => {}, + // Round up if the fractional part of the result is non-zero. + Rounding::Up => if rem_mul_upper % denom_upper > 0.into() { + // `rem * numer / denom` is less than `numer`, so this will not overflow. + rem_mul_div_inner = rem_mul_div_inner + 1.into(); + }, + // Round up if the fractional part of the result is greater than a half. An exact half is + // rounded down. + Rounding::Nearest => if rem_mul_upper % denom_upper > denom_upper / 2.into() { + // `rem * numer / denom` is less than `numer`, so this will not overflow. + rem_mul_div_inner = rem_mul_div_inner + 1.into(); + }, + } + rem_mul_div_inner.into() } macro_rules! implement_per_thing { - ($name:ident, $test_mod:ident, [$($test_units:tt),+], $max:tt, $type:ty, $upper_type:ty, $title:expr $(,)?) => { - /// A fixed point representation of a number between in the range [0, 1]. + ( + $name:ident, + $test_mod:ident, + [$($test_units:tt),+], + $max:tt, + $type:ty, + $upper_type:ty, + $title:expr $(,)? + ) => { + /// A fixed point representation of a number in the range [0, 1]. /// #[doc = $title] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] - #[derive(Encode, Decode, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, CompactAs)] + #[derive(Encode, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, + RuntimeDebug, CompactAs)] pub struct $name($type); impl PerThing for $name { type Inner = $type; + type Upper = $upper_type; - /// The accuracy of this type. const ACCURACY: Self::Inner = $max; - /// Nothing. - fn zero() -> Self { Self(0) } - - /// `true` if this is nothing. - fn is_zero(&self) -> bool { self.0 == 0 } - - /// Everything. - fn one() -> Self { Self($max) } - - /// Consume self and deconstruct into a raw numeric type. + /// Consume self and return the number of parts per thing. fn deconstruct(self) -> Self::Inner { self.0 } - /// From an explicitly defined number of parts per maximum of the type. - fn from_parts(parts: Self::Inner) -> Self { - Self([parts, $max][(parts > $max) as usize]) - } - - /// Converts a percent into `Self`. Equal to `x / 100`. - fn from_percent(x: Self::Inner) -> Self { - Self([x, 100][(x > 100) as usize] * ($max / 100)) - } + /// Build this type from a number of parts per thing. + fn from_parts(parts: Self::Inner) -> Self { Self(parts.min($max)) } - /// Return the product of multiplication of this value by itself. - fn square(self) -> Self { - // both can be safely casted and multiplied. - let p: $upper_type = self.0 as $upper_type * self.0 as $upper_type; - let q: $upper_type = <$upper_type>::from($max) * <$upper_type>::from($max); - Self::from_rational_approximation(p, q) - } - - /// Converts a fraction into `Self`. #[cfg(feature = "std")] - fn from_fraction(x: f64) -> Self { Self((x * ($max as f64)) as Self::Inner) } + fn from_fraction(x: f64) -> Self { + Self::from_parts((x * $max as f64) as Self::Inner) + } - /// Approximate the fraction `p/q` into a per-thing fraction. This will never overflow. - /// - /// The computation of this approximation is performed in the generic type `N`. Given - /// `M` as the data type that can hold the maximum value of this per-thing (e.g. u32 for - /// perbill), this can only work if `N == M` or `N: From + TryInto`. fn from_rational_approximation(p: N, q: N) -> Self - where N: Clone + Ord + From + TryInto + ops::Div + where N: Clone + Ord + From + TryInto + TryInto + + ops::Div + ops::Rem + ops::Add { + let div_ceil = |x: N, f: N| -> N { + let mut o = x.clone() / f.clone(); + let r = x.rem(f.clone()); + if r > N::from(0) { + o = o + N::from(1); + } + o + }; + // q cannot be zero. - let q = q.max((1 as Self::Inner).into()); + let q: N = q.max((1 as Self::Inner).into()); // p should not be bigger than q. - let p = p.min(q.clone()); + let p: N = p.min(q.clone()); - let factor = (q.clone() / $max.into()).max((1 as Self::Inner).into()); + let factor: N = div_ceil(q.clone(), $max.into()).max((1 as Self::Inner).into()); - // q cannot overflow: (q / (q/$max)) < 2 * $max. p < q hence p also cannot overflow. - // this implies that Self::Inner must be able to fit 2 * $max. - let q_reduce: Self::Inner = (q / factor.clone()) + // q cannot overflow: (q / (q/$max)) < $max. p < q hence p also cannot overflow. + let q_reduce: $type = (q.clone() / factor.clone()) .try_into() .map_err(|_| "Failed to convert") .expect( - "q / (q/$max) < (2 * $max). Macro prevents any type being created that \ + "q / ceil(q/$max) < $max. Macro prevents any type being created that \ does not satisfy this; qed" ); - let p_reduce: Self::Inner = (p / factor.clone()) + let p_reduce: $type = (p / factor) .try_into() .map_err(|_| "Failed to convert") .expect( - "q / (q/$max) < (2 * $max). Macro prevents any type being created that \ + "q / ceil(q/$max) < $max. Macro prevents any type being created that \ does not satisfy this; qed" ); @@ -159,11 +379,14 @@ macro_rules! implement_per_thing { } } - /// Implement const functions impl $name { /// From an explicitly defined number of parts per maximum of the type. /// /// This can be called at compile time. + // needed only for peru16. Since peru16 is the only type in which $max == + // $type::max_value(), rustc is being a smart-a** here by warning that the comparison + // is not needed. + #[allow(unused_comparisons)] pub const fn from_parts(parts: $type) -> Self { Self([parts, $max][(parts > $max) as usize]) } @@ -172,26 +395,110 @@ macro_rules! implement_per_thing { /// /// This can be created at compile time. pub const fn from_percent(x: $type) -> Self { - Self([x, 100][(x > 100) as usize] * ($max / 100)) + Self(([x, 100][(x > 100) as usize] as $upper_type * $max as $upper_type / 100) as $type) } - /// Everything. - /// - /// To avoid having to import `PerThing` when one needs to be used in test mocks. - #[cfg(feature = "std")] + /// See [`PerThing::one`]. pub fn one() -> Self { ::one() } + + /// See [`PerThing::is_one`]. + pub fn is_one(&self) -> bool { + PerThing::is_one(self) + } + + /// See [`PerThing::zero`]. + pub fn zero() -> Self { + ::zero() + } + + /// See [`PerThing::is_zero`]. + pub fn is_zero(&self) -> bool { + PerThing::is_zero(self) + } + + /// See [`PerThing::deconstruct`]. + pub fn deconstruct(self) -> $type { + PerThing::deconstruct(self) + } + + /// See [`PerThing::square`]. + pub fn square(self) -> Self { + PerThing::square(self) + } + + /// See [`PerThing::from_fraction`]. + #[cfg(feature = "std")] + pub fn from_fraction(x: f64) -> Self { + ::from_fraction(x) + } + + /// See [`PerThing::from_rational_approximation`]. + pub fn from_rational_approximation(p: N, q: N) -> Self + where N: Clone + Ord + From<$type> + TryInto<$type> + + TryInto<$upper_type> + ops::Div + ops::Rem + + ops::Add { + ::from_rational_approximation(p, q) + } + + /// See [`PerThing::mul_floor`]. + pub fn mul_floor(self, b: N) -> N + where N: Clone + From<$type> + UniqueSaturatedInto<$type> + + ops::Rem + ops::Div + ops::Mul + + ops::Add { + PerThing::mul_floor(self, b) + } + + /// See [`PerThing::mul_ceil`]. + pub fn mul_ceil(self, b: N) -> N + where N: Clone + From<$type> + UniqueSaturatedInto<$type> + + ops::Rem + ops::Div + ops::Mul + + ops::Add { + PerThing::mul_ceil(self, b) + } + + /// See [`PerThing::saturating_reciprocal_mul`]. + pub fn saturating_reciprocal_mul(self, b: N) -> N + where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + + ops::Div + ops::Mul + ops::Add + + Saturating { + PerThing::saturating_reciprocal_mul(self, b) + } + + /// See [`PerThing::saturating_reciprocal_mul_floor`]. + pub fn saturating_reciprocal_mul_floor(self, b: N) -> N + where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + + ops::Div + ops::Mul + ops::Add + + Saturating { + PerThing::saturating_reciprocal_mul_floor(self, b) + } + + /// See [`PerThing::saturating_reciprocal_mul_ceil`]. + pub fn saturating_reciprocal_mul_ceil(self, b: N) -> N + where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + + ops::Div + ops::Mul + ops::Add + + Saturating { + PerThing::saturating_reciprocal_mul_ceil(self, b) + } } impl Saturating for $name { + /// Saturating addition. Compute `self + rhs`, saturating at the numeric bounds instead of + /// overflowing. This operation is lossless if it does not saturate. fn saturating_add(self, rhs: Self) -> Self { // defensive-only: since `$max * 2 < $type::max_value()`, this can never overflow. Self::from_parts(self.0.saturating_add(rhs.0)) } + + /// Saturating subtraction. Compute `self - rhs`, saturating at the numeric bounds instead of + /// overflowing. This operation is lossless if it does not saturate. fn saturating_sub(self, rhs: Self) -> Self { Self::from_parts(self.0.saturating_sub(rhs.0)) } + + /// Saturating multiply. Compute `self * rhs`, saturating at the numeric bounds instead of + /// overflowing. This operation is lossy. fn saturating_mul(self, rhs: Self) -> Self { let a = self.0 as $upper_type; let b = rhs.0 as $upper_type; @@ -200,6 +507,53 @@ macro_rules! implement_per_thing { // This will always fit into $type. Self::from_parts(parts as $type) } + + /// Saturating exponentiation. Computes `self.pow(exp)`, saturating at the numeric + /// bounds instead of overflowing. This operation is lossy. + fn saturating_pow(self, exp: usize) -> Self { + if self.is_zero() || self.is_one() { + self + } else { + let p = <$name as PerThing>::Upper::from(self.deconstruct()); + let q = <$name as PerThing>::Upper::from(Self::ACCURACY); + let mut s = Self::one(); + for _ in 0..exp { + if s.is_zero() { + break; + } else { + // x^2 always fits in Self::Upper if x fits in Self::Inner. + // Verified by a test. + s = Self::from_rational_approximation( + <$name as PerThing>::Upper::from(s.deconstruct()) * p, + q * q, + ); + } + } + s + } + } + } + + impl codec::Decode for $name { + fn decode(input: &mut I) -> Result { + let inner = <$type as codec::Decode>::decode(input)?; + + if inner <= ::ACCURACY { + Ok(Self(inner)) + } else { + Err("Value is greater than allowed maximum!".into()) + } + } + } + + impl crate::traits::Bounded for $name { + fn min_value() -> Self { + ::zero() + } + + fn max_value() -> Self { + ::one() + } } impl ops::Div for $name { @@ -212,9 +566,9 @@ macro_rules! implement_per_thing { } } - /// Overflow-prune multiplication. + /// Non-overflow multiplication. /// - /// tailored to be used with a balance type. + /// This is tailored to be used with a balance type. impl ops::Mul for $name where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem @@ -222,36 +576,7 @@ macro_rules! implement_per_thing { { type Output = N; fn mul(self, b: N) -> Self::Output { - let maximum: N = $max.into(); - let upper_max: $upper_type = $max.into(); - let part: N = self.0.into(); - - let rem_multiplied_divided = { - let rem = b.clone().rem(maximum.clone()); - - // `rem_sized` is inferior to $max, thus it fits into $type. This is assured by - // a test. - let rem_sized = rem.saturated_into::<$type>(); - - // `self` and `rem_sized` are inferior to $max, thus the product is less than - // $max^2 and fits into $upper_type. This is assured by a test. - let rem_multiplied_upper = rem_sized as $upper_type * self.0 as $upper_type; - - // `rem_multiplied_upper` is less than $max^2 therefore divided by $max it fits - // in $type. remember that $type always fits $max. - let mut rem_multiplied_divided_sized = - (rem_multiplied_upper / upper_max) as $type; - // fix a tiny rounding error - if rem_multiplied_upper % upper_max > upper_max / 2 { - rem_multiplied_divided_sized += 1; - } - - // `rem_multiplied_divided_sized` is inferior to b, thus it can be converted - // back to N type - rem_multiplied_divided_sized.into() - }; - - (b / maximum) * part + rem_multiplied_divided + overflow_prune_mul::(b, self.deconstruct(), Rounding::Nearest) } } @@ -261,20 +586,25 @@ macro_rules! implement_per_thing { use super::{$name, Saturating, RuntimeDebug, PerThing}; use crate::traits::Zero; - #[test] fn macro_expanded_correctly() { - // needed for the `from_percent` to work. - assert!($max >= 100); - assert!($max % 100 == 0); + // needed for the `from_percent` to work. UPDATE: this is no longer needed; yet note + // that tests that use percentage or fractions such as $name::from_fraction(0.2) to + // create values will most likely be inaccurate when used with per_things that are + // not multiples of 100. + // assert!($max >= 100); + // assert!($max % 100 == 0); // needed for `from_rational_approximation` - assert!(2 * $max < <$type>::max_value()); + assert!(2 * ($max as $upper_type) < <$upper_type>::max_value()); assert!(<$upper_type>::from($max) < <$upper_type>::max_value()); // for something like percent they can be the same. assert!((<$type>::max_value() as $upper_type) <= <$upper_type>::max_value()); assert!(<$upper_type>::from($max).checked_mul($max.into()).is_some()); + + // make sure saturating_pow won't overflow the upper type + assert!(<$upper_type>::from($max) * <$upper_type>::from($max) < <$upper_type>::max_value()); } #[derive(Encode, Decode, PartialEq, Eq, RuntimeDebug)] @@ -298,7 +628,7 @@ macro_rules! implement_per_thing { (63, 1), (64, 2), (65, 2), - (<$type>::max_value(), <$type>::max_value().encode().len() + 1) + // (<$type>::max_value(), <$type>::max_value().encode().len() + 1) ]; for &(n, l) in &tests { let compact: codec::Compact<$name> = $name(n).into(); @@ -311,39 +641,100 @@ macro_rules! implement_per_thing { } } + #[test] + fn fail_on_invalid_encoded_value() { + let value = <$upper_type>::from($max) * 2; + let casted = value as $type; + let encoded = casted.encode(); + + // For types where `$max == $type::maximum()` we can not + if <$upper_type>::from(casted) == value { + assert_eq!( + $name::decode(&mut &encoded[..]), + Err("Value is greater than allowed maximum!".into()), + ); + } + } + #[test] fn per_thing_api_works() { // some really basic stuff assert_eq!($name::zero(), $name::from_parts(Zero::zero())); assert_eq!($name::one(), $name::from_parts($max)); assert_eq!($name::ACCURACY, $max); + assert_eq!($name::from_percent(0), $name::from_parts(Zero::zero())); assert_eq!($name::from_percent(10), $name::from_parts($max / 10)); assert_eq!($name::from_percent(100), $name::from_parts($max)); + assert_eq!($name::from_percent(200), $name::from_parts($max)); + + assert_eq!($name::from_fraction(0.0), $name::from_parts(Zero::zero())); + assert_eq!($name::from_fraction(0.1), $name::from_parts($max / 10)); + assert_eq!($name::from_fraction(1.0), $name::from_parts($max)); + } + + macro_rules! u256ify { + ($val:expr) => { + Into::::into($val) + }; } macro_rules! per_thing_mul_test { ($num_type:tt) => { // multiplication from all sort of from_percent assert_eq!( - $name::from_percent(100) * $num_type::max_value(), + $name::from_fraction(1.0) * $num_type::max_value(), $num_type::max_value() ); - assert_eq_error_rate!( - $name::from_percent(99) * $num_type::max_value(), - ((Into::::into($num_type::max_value()) * 99u32) / 100u32).as_u128() as $num_type, - 1, - ); - assert_eq!( - $name::from_percent(50) * $num_type::max_value(), - $num_type::max_value() / 2, - ); - assert_eq_error_rate!( - $name::from_percent(1) * $num_type::max_value(), - $num_type::max_value() / 100, - 1, - ); - assert_eq!($name::from_percent(0) * $num_type::max_value(), 0); + if $max % 100 == 0 { + assert_eq_error_rate!( + $name::from_percent(99) * $num_type::max_value(), + ((Into::::into($num_type::max_value()) * 99u32) / 100u32).as_u128() as $num_type, + 1, + ); + assert_eq!( + $name::from_fraction(0.5) * $num_type::max_value(), + $num_type::max_value() / 2, + ); + assert_eq_error_rate!( + $name::from_percent(1) * $num_type::max_value(), + $num_type::max_value() / 100, + 1, + ); + } else { + assert_eq!( + $name::from_fraction(0.99) * <$num_type>::max_value(), + ( + ( + u256ify!($name::from_fraction(0.99).0) * + u256ify!(<$num_type>::max_value()) / + u256ify!($max) + ).as_u128() + ) as $num_type, + ); + assert_eq!( + $name::from_fraction(0.50) * <$num_type>::max_value(), + ( + ( + u256ify!($name::from_fraction(0.50).0) * + u256ify!(<$num_type>::max_value()) / + u256ify!($max) + ).as_u128() + ) as $num_type, + ); + assert_eq!( + $name::from_fraction(0.01) * <$num_type>::max_value(), + ( + ( + u256ify!($name::from_fraction(0.01).0) * + u256ify!(<$num_type>::max_value()) / + u256ify!($max) + ).as_u128() + ) as $num_type, + ); + } + + assert_eq!($name::from_fraction(0.0) * $num_type::max_value(), 0); // // multiplication with bounds assert_eq!($name::one() * $num_type::max_value(), $num_type::max_value()); @@ -356,17 +747,20 @@ macro_rules! implement_per_thing { use primitive_types::U256; // accuracy test - assert_eq!($name::from_rational_approximation(1 as $type, 3) * 30 as $type, 10); + assert_eq!( + $name::from_rational_approximation(1 as $type, 3) * 30 as $type, + 10, + ); $(per_thing_mul_test!($test_units);)* } #[test] fn per_thing_mul_rounds_to_nearest_number() { - assert_eq!($name::from_percent(33) * 10u64, 3); - assert_eq!($name::from_percent(34) * 10u64, 3); - assert_eq!($name::from_percent(35) * 10u64, 3); - assert_eq!($name::from_percent(36) * 10u64, 4); + assert_eq!($name::from_fraction(0.33) * 10u64, 3); + assert_eq!($name::from_fraction(0.34) * 10u64, 3); + assert_eq!($name::from_fraction(0.35) * 10u64, 3); + assert_eq!($name::from_fraction(0.36) * 10u64, 4); } #[test] @@ -398,31 +792,32 @@ macro_rules! implement_per_thing { ); assert_eq!( $name::from_rational_approximation(1 as $num_type, 10), - $name::from_percent(10), + $name::from_fraction(0.10), ); assert_eq!( $name::from_rational_approximation(1 as $num_type, 4), - $name::from_percent(25), + $name::from_fraction(0.25), ); assert_eq!( $name::from_rational_approximation(1 as $num_type, 4), $name::from_rational_approximation(2 as $num_type, 8), ); // no accurate anymore but won't overflow. - assert_eq!( + assert_eq_error_rate!( $name::from_rational_approximation( $num_type::max_value() - 1, $num_type::max_value() - ), - $name::one(), + ).0 as $upper_type, + $name::one().0 as $upper_type, + 2, ); assert_eq_error_rate!( $name::from_rational_approximation( $num_type::max_value() / 3, $num_type::max_value() - ).0, - $name::from_parts($max / 3).0, - 2 + ).0 as $upper_type, + $name::from_parts($max / 3).0 as $upper_type, + 2, ); assert_eq!( $name::from_rational_approximation(1, $num_type::max_value()), @@ -436,13 +831,14 @@ macro_rules! implement_per_thing { // This is just to make sure something like Percent which _might_ get built from a // u8 does not overflow in the context of this test. let max_value = <$upper_type>::from($max); + // almost at the edge assert_eq!( - $name::from_rational_approximation($max - 1, $max + 1), + $name::from_rational_approximation(max_value - 1, max_value + 1), $name::from_parts($max - 2), ); assert_eq!( - $name::from_rational_approximation(1, $max-1), + $name::from_rational_approximation(1, $max - 1), $name::from_parts(1), ); assert_eq!( @@ -450,76 +846,83 @@ macro_rules! implement_per_thing { $name::from_parts(1), ); assert_eq!( - $name::from_rational_approximation(2, 2 * $max - 1), + $name::from_rational_approximation(2, 2 * max_value - 1), $name::from_parts(1), ); assert_eq!( - $name::from_rational_approximation(1, $max+1), + $name::from_rational_approximation(1, max_value + 1), $name::zero(), ); assert_eq!( $name::from_rational_approximation(3 * max_value / 2, 3 * max_value), - $name::from_percent(50), + $name::from_fraction(0.5), ); + $(per_thing_from_rationale_approx_test!($test_units);)* } #[test] fn per_things_mul_operates_in_output_type() { - // assert_eq!($name::from_percent(50) * 100u32, 50u32); - assert_eq!($name::from_percent(50) * 100u64, 50u64); - assert_eq!($name::from_percent(50) * 100u128, 50u128); + // assert_eq!($name::from_fraction(0.5) * 100u32, 50u32); + assert_eq!($name::from_fraction(0.5) * 100u64, 50u64); + assert_eq!($name::from_fraction(0.5) * 100u128, 50u128); } #[test] fn per_thing_saturating_op_works() { - assert_eq!( - $name::from_percent(50).saturating_add($name::from_percent(40)), - $name::from_percent(90) + assert_eq_error_rate!( + $name::from_fraction(0.5).saturating_add($name::from_fraction(0.4)).0 as $upper_type, + $name::from_fraction(0.9).0 as $upper_type, + 2, ); - assert_eq!( - $name::from_percent(50).saturating_add($name::from_percent(50)), - $name::from_percent(100) + assert_eq_error_rate!( + $name::from_fraction(0.5).saturating_add($name::from_fraction(0.5)).0 as $upper_type, + $name::one().0 as $upper_type, + 2, ); assert_eq!( - $name::from_percent(60).saturating_add($name::from_percent(50)), - $name::from_percent(100) + $name::from_fraction(0.6).saturating_add($name::from_fraction(0.5)), + $name::one(), ); - assert_eq!( - $name::from_percent(60).saturating_sub($name::from_percent(50)), - $name::from_percent(10) + assert_eq_error_rate!( + $name::from_fraction(0.6).saturating_sub($name::from_fraction(0.5)).0 as $upper_type, + $name::from_fraction(0.1).0 as $upper_type, + 2, ); assert_eq!( - $name::from_percent(60).saturating_sub($name::from_percent(60)), - $name::from_percent(0) + $name::from_fraction(0.6).saturating_sub($name::from_fraction(0.6)), + $name::from_fraction(0.0), ); assert_eq!( - $name::from_percent(60).saturating_sub($name::from_percent(70)), - $name::from_percent(0) + $name::from_fraction(0.6).saturating_sub($name::from_fraction(0.7)), + $name::from_fraction(0.0), ); - assert_eq!( - $name::from_percent(50).saturating_mul($name::from_percent(50)), - $name::from_percent(25) + assert_eq_error_rate!( + $name::from_fraction(0.5).saturating_mul($name::from_fraction(0.5)).0 as $upper_type, + $name::from_fraction(0.25).0 as $upper_type, + 2, ); - assert_eq!( - $name::from_percent(20).saturating_mul($name::from_percent(20)), - $name::from_percent(4) + assert_eq_error_rate!( + $name::from_fraction(0.2).saturating_mul($name::from_fraction(0.2)).0 as $upper_type, + $name::from_fraction(0.04).0 as $upper_type, + 2, ); - assert_eq!( - $name::from_percent(10).saturating_mul($name::from_percent(10)), - $name::from_percent(1) + assert_eq_error_rate!( + $name::from_fraction(0.1).saturating_mul($name::from_fraction(0.1)).0 as $upper_type, + $name::from_fraction(0.01).0 as $upper_type, + 1, ); } #[test] fn per_thing_square_works() { - assert_eq!($name::from_percent(100).square(), $name::from_percent(100)); - assert_eq!($name::from_percent(50).square(), $name::from_percent(25)); - assert_eq!($name::from_percent(10).square(), $name::from_percent(1)); + assert_eq!($name::from_fraction(1.0).square(), $name::from_fraction(1.0)); + assert_eq!($name::from_fraction(0.5).square(), $name::from_fraction(0.25)); + assert_eq!($name::from_fraction(0.1).square(), $name::from_fraction(0.01)); assert_eq!( - $name::from_percent(2).square(), + $name::from_fraction(0.02).square(), $name::from_parts((4 * <$upper_type>::from($max) / 100 / 100) as $type) ); } @@ -527,22 +930,203 @@ macro_rules! implement_per_thing { #[test] fn per_things_div_works() { // normal - assert_eq!($name::from_percent(10) / $name::from_percent(20), - $name::from_percent(50) + assert_eq_error_rate!( + ($name::from_fraction(0.1) / $name::from_fraction(0.20)).0 as $upper_type, + $name::from_fraction(0.50).0 as $upper_type, + 2, ); - assert_eq!($name::from_percent(10) / $name::from_percent(10), - $name::from_percent(100) + assert_eq_error_rate!( + ($name::from_fraction(0.1) / $name::from_fraction(0.10)).0 as $upper_type, + $name::from_fraction(1.0).0 as $upper_type, + 2, ); - assert_eq!($name::from_percent(10) / $name::from_percent(0), - $name::from_percent(100) + assert_eq_error_rate!( + ($name::from_fraction(0.1) / $name::from_fraction(0.0)).0 as $upper_type, + $name::from_fraction(1.0).0 as $upper_type, + 2, ); // will not overflow - assert_eq!($name::from_percent(10) / $name::from_percent(5), - $name::from_percent(100) + assert_eq_error_rate!( + ($name::from_fraction(0.10) / $name::from_fraction(0.05)).0 as $upper_type, + $name::from_fraction(1.0).0 as $upper_type, + 2, + ); + assert_eq_error_rate!( + ($name::from_fraction(1.0) / $name::from_fraction(0.5)).0 as $upper_type, + $name::from_fraction(1.0).0 as $upper_type, + 2, + ); + } + + #[test] + fn saturating_pow_works() { + // x^0 == 1 + assert_eq!( + $name::from_parts($max / 2).saturating_pow(0), + $name::from_parts($max), + ); + + // x^1 == x + assert_eq!( + $name::from_parts($max / 2).saturating_pow(1), + $name::from_parts($max / 2), + ); + + // x^2 + assert_eq!( + $name::from_parts($max / 2).saturating_pow(2), + $name::from_parts($max / 2).square(), + ); + + // x^3 + assert_eq!( + $name::from_parts($max / 2).saturating_pow(3), + $name::from_parts($max / 8), + ); + + // 0^n == 0 + assert_eq!( + $name::from_parts(0).saturating_pow(3), + $name::from_parts(0), ); - assert_eq!($name::from_percent(100) / $name::from_percent(50), - $name::from_percent(100) + + // 1^n == 1 + assert_eq!( + $name::from_parts($max).saturating_pow(3), + $name::from_parts($max), + ); + + // (x < 1)^inf == 0 (where 2.pow(31) ~ inf) + assert_eq!( + $name::from_parts($max / 2).saturating_pow(2usize.pow(31)), + $name::from_parts(0), + ); + } + + #[test] + fn saturating_reciprocal_mul_works() { + // divide by 1 + assert_eq!( + $name::from_parts($max).saturating_reciprocal_mul(<$type>::from(10u8)), + 10, + ); + // divide by 1/2 + assert_eq!( + $name::from_parts($max / 2).saturating_reciprocal_mul(<$type>::from(10u8)), + 20, + ); + // saturate + assert_eq!( + $name::from_parts(1).saturating_reciprocal_mul($max), + <$type>::max_value(), + ); + // round to nearest + assert_eq!( + $name::from_percent(60).saturating_reciprocal_mul(<$type>::from(10u8)), + 17, + ); + // round down + assert_eq!( + $name::from_percent(60).saturating_reciprocal_mul_floor(<$type>::from(10u8)), + 16, + ); + // round to nearest + assert_eq!( + $name::from_percent(61).saturating_reciprocal_mul(<$type>::from(10u8)), + 16, + ); + // round up + assert_eq!( + $name::from_percent(61).saturating_reciprocal_mul_ceil(<$type>::from(10u8)), + 17, + ); + } + + #[test] + fn saturating_truncating_mul_works() { + assert_eq!( + $name::from_percent(49).mul_floor(10 as $type), + 4, + ); + let a: $upper_type = $name::from_percent(50).mul_floor(($max as $upper_type).pow(2)); + let b: $upper_type = ($max as $upper_type).pow(2) / 2; + if $max % 2 == 0 { + assert_eq!(a, b); + } else { + // difference should be less that 1%, IE less than the error in `from_percent` + assert!(b - a < ($max as $upper_type).pow(2) / 100 as $upper_type); + } + } + + #[test] + fn rational_mul_correction_works() { + assert_eq!( + super::rational_mul_correction::<$type, $name>( + <$type>::max_value(), + <$type>::max_value(), + <$type>::max_value(), + super::Rounding::Nearest, + ), + 0, + ); + assert_eq!( + super::rational_mul_correction::<$type, $name>( + <$type>::max_value() - 1, + <$type>::max_value(), + <$type>::max_value(), + super::Rounding::Nearest, + ), + <$type>::max_value() - 1, + ); + assert_eq!( + super::rational_mul_correction::<$upper_type, $name>( + ((<$type>::max_value() - 1) as $upper_type).pow(2), + <$type>::max_value(), + <$type>::max_value(), + super::Rounding::Nearest, + ), + 1, + ); + // ((max^2 - 1) % max) * max / max == max - 1 + assert_eq!( + super::rational_mul_correction::<$upper_type, $name>( + (<$type>::max_value() as $upper_type).pow(2) - 1, + <$type>::max_value(), + <$type>::max_value(), + super::Rounding::Nearest, + ), + (<$type>::max_value() - 1).into(), + ); + // (max % 2) * max / 2 == max / 2 + assert_eq!( + super::rational_mul_correction::<$upper_type, $name>( + (<$type>::max_value() as $upper_type).pow(2), + <$type>::max_value(), + 2 as $type, + super::Rounding::Nearest, + ), + <$type>::max_value() as $upper_type / 2, + ); + // ((max^2 - 1) % max) * 2 / max == 2 (rounded up) + assert_eq!( + super::rational_mul_correction::<$upper_type, $name>( + (<$type>::max_value() as $upper_type).pow(2) - 1, + 2 as $type, + <$type>::max_value(), + super::Rounding::Nearest, + ), + 2, + ); + // ((max^2 - 1) % max) * 2 / max == 1 (rounded down) + assert_eq!( + super::rational_mul_correction::<$upper_type, $name>( + (<$type>::max_value() as $upper_type).pow(2) - 1, + 2 as $type, + <$type>::max_value(), + super::Rounding::Down, + ), + 1, ); } } @@ -558,6 +1142,15 @@ implement_per_thing!( u16, "_Percent_", ); +implement_per_thing!( + PerU16, + test_peru16, + [u32, u64, u128], + 65535_u16, + u16, + u32, + "_Parts per 65535_", +); implement_per_thing!( Permill, test_permill, diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index 75adf0e1363dd6c460707d06a4852f2f39c4fbe2..23f8f23f0bd77b783dfc1908b0efc6354f9fc4a8 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -21,7 +21,7 @@ use codec::HasCompact; pub use integer_sqrt::IntegerSquareRoot; pub use num_traits::{ Zero, One, Bounded, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv, - CheckedShl, CheckedShr + CheckedShl, CheckedShr, checked_pow }; use sp_std::ops::{ Add, Sub, Mul, Div, Rem, AddAssign, SubAssign, MulAssign, DivAssign, @@ -104,29 +104,41 @@ impl + Sized> UniqueSaturatedInto for S { } } -/// Simple trait to use checked mul and max value to give a saturated mul operation over -/// supported types. +/// Saturating arithmetic operations, returning maximum or minimum values instead of overflowing. pub trait Saturating { - /// Saturated addition - if the product can't fit in the type then just use max-value. - fn saturating_add(self, o: Self) -> Self; - - /// Saturated subtraction - if the product can't fit in the type then just use max-value. - fn saturating_sub(self, o: Self) -> Self; - - /// Saturated multiply - if the product can't fit in the type then just use max-value. - fn saturating_mul(self, o: Self) -> Self; + /// Saturating addition. Compute `self + rhs`, saturating at the numeric bounds instead of + /// overflowing. + fn saturating_add(self, rhs: Self) -> Self; + + /// Saturating subtraction. Compute `self - rhs`, saturating at the numeric bounds instead of + /// overflowing. + fn saturating_sub(self, rhs: Self) -> Self; + + /// Saturating multiply. Compute `self * rhs`, saturating at the numeric bounds instead of + /// overflowing. + fn saturating_mul(self, rhs: Self) -> Self; + + /// Saturating exponentiation. Compute `self.pow(exp)`, saturating at the numeric bounds + /// instead of overflowing. + fn saturating_pow(self, exp: usize) -> Self; } -impl Saturating for T { +impl Saturating for T { fn saturating_add(self, o: Self) -> Self { ::saturating_add(self, o) } + fn saturating_sub(self, o: Self) -> Self { ::saturating_sub(self, o) } + fn saturating_mul(self, o: Self) -> Self { self.checked_mul(&o).unwrap_or_else(Bounded::max_value) } + + fn saturating_pow(self, exp: usize) -> Self { + checked_pow(self, exp).unwrap_or_else(Bounded::max_value) + } } /// Convenience type to work around the highly unergonomic syntax needed diff --git a/primitives/authority-discovery/Cargo.toml b/primitives/authority-discovery/Cargo.toml index 3f40bb1b7580051013c88b491058d064273e1888..f37b67fab1ce8d476fef66656b3c5013701edbf9 100644 --- a/primitives/authority-discovery/Cargo.toml +++ b/primitives/authority-discovery/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", default-features = false, version = "1.2.0" } +codec = { package = "parity-scale-codec", default-features = false, version = "1.3.0" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } @@ -24,3 +24,6 @@ std = [ "sp-api/std", "sp-runtime/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/authority-discovery/src/lib.rs b/primitives/authority-discovery/src/lib.rs index 8926825525976f9d284bc1b6969ddfde936aff99..68680ad75946545f93d8d0c5ef5c3004a3623efc 100644 --- a/primitives/authority-discovery/src/lib.rs +++ b/primitives/authority-discovery/src/lib.rs @@ -21,8 +21,25 @@ use sp_std::vec::Vec; mod app { - use sp_application_crypto::{app_crypto, key_types::AUTHORITY_DISCOVERY, sr25519}; + use sp_application_crypto::{ + CryptoTypePublicPair, + key_types::AUTHORITY_DISCOVERY, + Public as _, + app_crypto, + sr25519}; app_crypto!(sr25519, AUTHORITY_DISCOVERY); + + impl From for CryptoTypePublicPair { + fn from(key: Public) -> Self { + (&key).into() + } + } + + impl From<&Public> for CryptoTypePublicPair { + fn from(key: &Public) -> Self { + CryptoTypePublicPair(sr25519::CRYPTO_ID, key.to_raw_vec()) + } + } } sp_application_crypto::with_pair! { diff --git a/primitives/authorship/Cargo.toml b/primitives/authorship/Cargo.toml index a69ce303bc08508b655a842e4b4109c345d5cf24..a4d5aa03c2382c3c93f278bca825c7ca08688951 100644 --- a/primitives/authorship/Cargo.toml +++ b/primitives/authorship/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/paritytech/substrate/" sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] @@ -22,3 +22,6 @@ std = [ "sp-inherents/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/block-builder/Cargo.toml b/primitives/block-builder/Cargo.toml index 13c8062aded6243362544001e73684eb4b1fb539..df33b2c955ff096b1bd861769d1dd5c56497a867 100644 --- a/primitives/block-builder/Cargo.toml +++ b/primitives/block-builder/Cargo.toml @@ -12,7 +12,7 @@ description = "The block builder runtime api." sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } [features] @@ -24,3 +24,6 @@ std = [ "sp-api/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/blockchain/Cargo.toml b/primitives/blockchain/Cargo.toml index 097f9048e57c8bfceb4771135fb2b8fa037a64c1..49e6baead1fb82a919b3d7ca40fe0a6045ee0524 100644 --- a/primitives/blockchain/Cargo.toml +++ b/primitives/blockchain/Cargo.toml @@ -15,8 +15,11 @@ log = "0.4.8" lru = "0.4.0" parking_lot = "0.10.0" derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-consensus = { version = "0.8.0-alpha.5", path = "../consensus/common" } sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } sp-block-builder = { version = "2.0.0-alpha.5", path = "../block-builder" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../state-machine" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index 35cac1e481120e2a88f9dafeebfc26f6e7287326..e92dfd8c98e89397ee2e3d10e6f85e71b0c5ac00 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -254,6 +254,8 @@ pub struct Info { pub finalized_hash: Block::Hash, /// Last finalized block number. pub finalized_number: <::Header as HeaderT>::Number, + /// Number of concurrent leave forks. + pub number_leaves: usize } /// Block status. diff --git a/primitives/consensus/aura/Cargo.toml b/primitives/consensus/aura/Cargo.toml index af25bfaba5fbcf93898201111148ace41b2baefa..99ce51a229c72e045e97b92f9e8c8f8d3aff2513 100644 --- a/primitives/consensus/aura/Cargo.toml +++ b/primitives/consensus/aura/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../api" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } @@ -28,3 +28,6 @@ std = [ "sp-inherents/std", "sp-timestamp/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/babe/Cargo.toml b/primitives/consensus/babe/Cargo.toml index d0360406430ee26e0cf86b15939e2e26f80d4f01..195a54a5910c419957f053a6f928aa369b467110 100644 --- a/primitives/consensus/babe/Cargo.toml +++ b/primitives/consensus/babe/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../api" } sp-consensus = { version = "0.8.0-alpha.5", optional = true, path = "../common" } @@ -32,3 +32,6 @@ std = [ "sp-runtime/std", "sp-timestamp/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/babe/src/digests.rs b/primitives/consensus/babe/src/digests.rs index fca0c3a7b31a65f5476db11d4ed8f12efe445533..6079aa88c87492538fbf3caefdd3775d4cdd9292 100644 --- a/primitives/consensus/babe/src/digests.rs +++ b/primitives/consensus/babe/src/digests.rs @@ -27,40 +27,67 @@ use codec::{Decode, Encode}; #[cfg(feature = "std")] use codec::Codec; use sp_std::vec::Vec; +use sp_runtime::RuntimeDebug; use sp_consensus_vrf::schnorrkel::{self, Randomness}; #[cfg(feature = "std")] use sp_consensus_vrf::schnorrkel::SignatureError; +/// Raw BABE primary slot assignment pre-digest. +#[derive(Clone, RuntimeDebug, Encode, Decode)] +pub struct RawPrimaryPreDigest { + /// Authority index + pub authority_index: super::AuthorityIndex, + /// Slot number + pub slot_number: SlotNumber, + /// VRF output + pub vrf_output: VRFOutput, + /// VRF proof + pub vrf_proof: VRFProof, +} + +#[cfg(feature = "std")] +/// BABE primary slot assignment pre-digest for std environment. +pub type PrimaryPreDigest = RawPrimaryPreDigest; + +#[cfg(feature = "std")] +impl TryFrom for PrimaryPreDigest { + type Error = SignatureError; + + fn try_from(raw: RawPrimaryPreDigest) -> Result { + Ok(PrimaryPreDigest { + authority_index: raw.authority_index, + slot_number: raw.slot_number, + vrf_output: raw.vrf_output.try_into()?, + vrf_proof: raw.vrf_proof.try_into()?, + }) + } +} + +/// BABE secondary slot assignment pre-digest. +#[derive(Clone, RuntimeDebug, Encode, Decode)] +pub struct SecondaryPreDigest { + /// Authority index + /// + /// This is not strictly-speaking necessary, since the secondary slots + /// are assigned based on slot number and epoch randomness. But including + /// it makes things easier for higher-level users of the chain data to + /// be aware of the author of a secondary-slot block. + pub authority_index: super::AuthorityIndex, + /// Slot number + pub slot_number: SlotNumber, +} + /// A BABE pre-runtime digest. This contains all data required to validate a /// block and for the BABE runtime module. Slots can be assigned to a primary /// (VRF based) and to a secondary (slot number based). -#[derive(Clone, Debug, Encode, Decode)] +#[derive(Clone, RuntimeDebug, Encode, Decode)] pub enum RawPreDigest { /// A primary VRF-based slot assignment. #[codec(index = "1")] - Primary { - /// Authority index - authority_index: super::AuthorityIndex, - /// Slot number - slot_number: SlotNumber, - /// VRF output - vrf_output: VRFOutput, - /// VRF proof - vrf_proof: VRFProof, - }, + Primary(RawPrimaryPreDigest), /// A secondary deterministic slot assignment. #[codec(index = "2")] - Secondary { - /// Authority index - /// - /// This is not strictly-speaking necessary, since the secondary slots - /// are assigned based on slot number and epoch randomness. But including - /// it makes things easier for higher-level users of the chain data to - /// be aware of the author of a secondary-slot block. - authority_index: super::AuthorityIndex, - /// Slot number - slot_number: SlotNumber, - }, + Secondary(SecondaryPreDigest), } #[cfg(feature = "std")] @@ -71,16 +98,16 @@ impl RawPreDigest { /// Returns the slot number of the pre digest. pub fn authority_index(&self) -> AuthorityIndex { match self { - RawPreDigest::Primary { authority_index, .. } => *authority_index, - RawPreDigest::Secondary { authority_index, .. } => *authority_index, + RawPreDigest::Primary(primary) => primary.authority_index, + RawPreDigest::Secondary(secondary) => secondary.authority_index, } } /// Returns the slot number of the pre digest. pub fn slot_number(&self) -> SlotNumber { match self { - RawPreDigest::Primary { slot_number, .. } => *slot_number, - RawPreDigest::Secondary { slot_number, .. } => *slot_number, + RawPreDigest::Primary(primary) => primary.slot_number, + RawPreDigest::Secondary(secondary) => secondary.slot_number, } } @@ -88,8 +115,8 @@ impl RawPreDigest { /// of the chain. pub fn added_weight(&self) -> crate::BabeBlockWeight { match self { - RawPreDigest::Primary { .. } => 1, - RawPreDigest::Secondary { .. } => 0, + RawPreDigest::Primary(_) => 1, + RawPreDigest::Secondary(_) => 0, } } } @@ -100,25 +127,15 @@ impl TryFrom for PreDigest { fn try_from(raw: RawPreDigest) -> Result { Ok(match raw { - RawPreDigest::Primary { authority_index, slot_number, vrf_output, vrf_proof } => - RawPreDigest::Primary { - authority_index, - slot_number, - vrf_output: vrf_output.try_into()?, - vrf_proof: vrf_proof.try_into()?, - }, - RawPreDigest::Secondary { authority_index, slot_number } => - RawPreDigest::Secondary { - authority_index, - slot_number, - } + RawPreDigest::Primary(primary) => PreDigest::Primary(primary.try_into()?), + RawPreDigest::Secondary(secondary) => PreDigest::Secondary(secondary), }) } } /// Information about the next epoch. This is broadcast in the first block /// of the epoch. -#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, sp_runtime::RuntimeDebug)] +#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, RuntimeDebug)] pub struct NextEpochDescriptor { /// The authorities. pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml index db66fd0d570b61c92c11b68e98a13b9b4c36269f..4734cde694261bd3d09c4eb688619f8d208c07eb 100644 --- a/primitives/consensus/common/Cargo.toml +++ b/primitives/consensus/common/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/sp-consensus/" [dependencies] derive_more = "0.99.2" -libp2p = { version = "0.16.2", default-features = false } +libp2p = { version = "0.17.0", default-features = false } log = "0.4.8" sp-core = { path= "../../core" , version = "2.0.0-alpha.5"} sp-inherents = { version = "2.0.0-alpha.5", path = "../../inherents" } @@ -23,7 +23,8 @@ futures-diagnose = "1.0" sp-std = { version = "2.0.0-alpha.5", path = "../../std" } sp-version = { version = "2.0.0-alpha.5", path = "../../version" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +sp-utils = { version = "2.0.0-alpha.5", path = "../../utils" } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" serde = { version = "1.0", features = ["derive"] } @@ -32,3 +33,6 @@ sp-test-primitives = { version = "2.0.0-dev", path = "../../test-primitives" } [features] default = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index 0d1aed7fb1c72e4225fab3c48cba9faade48c20c..024e473849349f416ce0d84a87222f2fb57bd88b 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -15,10 +15,11 @@ // along with Substrate. If not, see . use std::{mem, pin::Pin, time::Duration, marker::PhantomData, sync::Arc}; -use futures::{prelude::*, channel::mpsc, task::Context, task::Poll}; +use futures::{prelude::*, task::Context, task::Poll}; use futures_timer::Delay; use parking_lot::{Mutex, Condvar}; use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}}; +use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded}; use crate::block_import::BlockOrigin; use crate::import_queue::{ @@ -32,7 +33,7 @@ use crate::import_queue::{ /// task, with plugable verification. pub struct BasicQueue { /// Channel to send messages to the background task. - sender: mpsc::UnboundedSender>, + sender: TracingUnboundedSender>, /// Results coming from the worker task. result_port: BufferedLinkReceiver, /// If it isn't possible to spawn the future in `future_to_spawn` (which is notably the case in @@ -195,8 +196,8 @@ impl BlockImportWorker { block_import: BoxBlockImport, justification_import: Option>, finality_proof_import: Option>, - ) -> (impl Future + Send, mpsc::UnboundedSender>) { - let (sender, mut port) = mpsc::unbounded(); + ) -> (impl Future + Send, TracingUnboundedSender>) { + let (sender, mut port) = tracing_unbounded("mpsc_block_import_worker"); let mut worker = BlockImportWorker { result_sender, diff --git a/primitives/consensus/common/src/import_queue/buffered_link.rs b/primitives/consensus/common/src/import_queue/buffered_link.rs index d0f6c87951354f602f5400fa28b95a87673cbef7..ea77fc97f0e8a28e889ea8c13c1644a2f968a734 100644 --- a/primitives/consensus/common/src/import_queue/buffered_link.rs +++ b/primitives/consensus/common/src/import_queue/buffered_link.rs @@ -37,8 +37,9 @@ //! ``` //! -use futures::{prelude::*, channel::mpsc}; +use futures::prelude::*; use sp_runtime::traits::{Block as BlockT, NumberFor}; +use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded}; use std::{pin::Pin, task::Context, task::Poll}; use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError}; @@ -46,7 +47,7 @@ use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError}; /// can be used to buffer commands, and the receiver can be used to poll said commands and transfer /// them to another link. pub fn buffered_link() -> (BufferedLinkSender, BufferedLinkReceiver) { - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("mpsc_buffered_link"); let tx = BufferedLinkSender { tx }; let rx = BufferedLinkReceiver { rx }; (tx, rx) @@ -54,7 +55,7 @@ pub fn buffered_link() -> (BufferedLinkSender, BufferedLinkReceive /// See [`buffered_link`]. pub struct BufferedLinkSender { - tx: mpsc::UnboundedSender>, + tx: TracingUnboundedSender>, } impl BufferedLinkSender { @@ -125,7 +126,7 @@ impl Link for BufferedLinkSender { /// See [`buffered_link`]. pub struct BufferedLinkReceiver { - rx: mpsc::UnboundedReceiver>, + rx: TracingUnboundedReceiver>, } impl BufferedLinkReceiver { diff --git a/primitives/consensus/pow/Cargo.toml b/primitives/consensus/pow/Cargo.toml index 38be986d1dee9f53f871cf2772fed7d287752ecf..5ca60bb2155590fa34e1368ee1758ce7639871aa 100644 --- a/primitives/consensus/pow/Cargo.toml +++ b/primitives/consensus/pow/Cargo.toml @@ -13,7 +13,7 @@ sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../ap sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] default = ["std"] @@ -24,3 +24,6 @@ std = [ "sp-core/std", "codec/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/vrf/Cargo.toml b/primitives/consensus/vrf/Cargo.toml index 8fedd3f0e1561345aa741301ae7791913455f708..cf194ec38b50ac75b638044936ceca73172309c8 100644 --- a/primitives/consensus/vrf/Cargo.toml +++ b/primitives/consensus/vrf/Cargo.toml @@ -24,3 +24,6 @@ std = [ "sp-core/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/vrf/src/schnorrkel.rs b/primitives/consensus/vrf/src/schnorrkel.rs index e4ae68ced41902a13dc8ec752dd946ef314293db..265572dbdaee78dc4848f075bf22a690ea743a4e 100644 --- a/primitives/consensus/vrf/src/schnorrkel.rs +++ b/primitives/consensus/vrf/src/schnorrkel.rs @@ -17,7 +17,6 @@ //! Schnorrkel-based VRF. use codec::{Encode, Decode}; -use sp_core::U512; use sp_runtime::RuntimeDebug; use sp_std::ops::{Deref, DerefMut}; #[cfg(feature = "std")] @@ -26,6 +25,8 @@ use std::convert::TryFrom; use codec::EncodeLike; #[cfg(feature = "std")] use schnorrkel::errors::MultiSignatureStage; +#[cfg(feature = "std")] +use sp_core::U512; #[cfg(feature = "std")] pub use schnorrkel::{SignatureError, vrf::{VRF_PROOF_LENGTH, VRF_OUTPUT_LENGTH}}; diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 5def0e8b3feb08814afb3e12574de3a51a80548a..7708188ea4bf230e40a06814e5ac0773773da173 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/sp-core" [dependencies] sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } log = { version = "0.4.8", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } byteorder = { version = "1.3.2", default-features = false } @@ -119,3 +119,6 @@ full_crypto = [ "libsecp256k1", "sp-runtime-interface/disable_target_static_assertions", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index 259c3c5f9a7043425aaccbe708bcea4d3c0df225..79a36b2ad2353887aa8c581de552e34f2fb34d3e 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -18,8 +18,10 @@ //! Cryptographic utilities. // end::description[] +use crate::{sr25519, ed25519}; use sp_std::hash::Hash; use sp_std::vec::Vec; +use sp_std::str; #[cfg(feature = "std")] use sp_std::convert::TryInto; use sp_std::convert::TryFrom; @@ -32,7 +34,8 @@ use codec::{Encode, Decode}; use regex::Regex; #[cfg(feature = "std")] use base58::{FromBase58, ToBase58}; - +#[cfg(feature = "std")] +use crate::hexdisplay::HexDisplay; use zeroize::Zeroize; #[doc(hidden)] pub use sp_std::ops::Deref; @@ -456,6 +459,8 @@ ss58_address_format!( (16, "kulupu", "Kulupu mainnet, standard account (*25519).") DarwiniaAccount => (18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).") + RobonomicsAccount => + (32, "robonomics", "Any Robonomics network standard account (*25519).") CentrifugeAccount => (36, "centrifuge", "Centrifuge Chain mainnet, standard account (*25519).") SubstrateAccount => @@ -538,7 +543,9 @@ impl + AsRef<[u8]> + Default + Derive> Ss58Codec for T { } /// Trait suitable for typical cryptographic PKI key public type. -pub trait Public: AsRef<[u8]> + AsMut<[u8]> + Default + Derive + CryptoType + PartialEq + Eq + Clone + Send + Sync { +pub trait Public: + AsRef<[u8]> + AsMut<[u8]> + Default + Derive + CryptoType + PartialEq + Eq + Clone + Send + Sync +{ /// A new instance from the given slice. /// /// NOTE: No checking goes on to ensure this is a real public key. Only use it if @@ -554,6 +561,7 @@ pub trait Public: AsRef<[u8]> + AsMut<[u8]> + Default + Derive + CryptoType + Pa /// An opaque 32-byte cryptographic identifier. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Default, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Hash))] pub struct AccountId32([u8; 32]); impl UncheckedFrom for AccountId32 { @@ -614,6 +622,18 @@ impl From for [u8; 32] { } } +impl From for AccountId32 { + fn from(k: sr25519::Public) -> Self { + k.0.into() + } +} + +impl From for AccountId32 { + fn from(k: ed25519::Public) -> Self { + k.0.into() + } +} + #[cfg(feature = "std")] impl std::fmt::Display for AccountId32 { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { @@ -941,6 +961,27 @@ impl<'a> TryFrom<&'a str> for KeyTypeId { } } +/// An identifier for a specific cryptographic algorithm used by a key pair +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] +pub struct CryptoTypeId(pub [u8; 4]); + +/// A type alias of CryptoTypeId & a public key +#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] +pub struct CryptoTypePublicPair(pub CryptoTypeId, pub Vec); + +#[cfg(feature = "std")] +impl sp_std::fmt::Display for CryptoTypePublicPair { + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + let id = match str::from_utf8(&(self.0).0[..]) { + Ok(id) => id.to_string(), + Err(_) => { + format!("{:#?}", self.0) + } + }; + write!(f, "{}-{}", id, HexDisplay::from(&self.1)) + } +} + /// Known key types; this also functions as a global registry of key types for projects wishing to /// avoid collisions with each other. /// @@ -961,6 +1002,8 @@ pub mod key_types { pub const IM_ONLINE: KeyTypeId = KeyTypeId(*b"imon"); /// Key type for AuthorityDiscovery module, built-in. pub const AUTHORITY_DISCOVERY: KeyTypeId = KeyTypeId(*b"audi"); + /// Key type for staking, built-in. + pub const STAKING: KeyTypeId = KeyTypeId(*b"stak"); /// A key type ID useful for tests. pub const DUMMY: KeyTypeId = KeyTypeId(*b"dumy"); } diff --git a/primitives/core/src/ecdsa.rs b/primitives/core/src/ecdsa.rs index 548ae49b55487d31779a150909456beb74be04c8..8a45157844f3af20ce5ded4ce3dc28f6008a5983 100644 --- a/primitives/core/src/ecdsa.rs +++ b/primitives/core/src/ecdsa.rs @@ -36,10 +36,13 @@ use crate::{hashing::blake2_256, crypto::{Pair as TraitPair, DeriveJunction, Sec use crate::crypto::Ss58Codec; #[cfg(feature = "std")] use serde::{de, Serializer, Serialize, Deserializer, Deserialize}; -use crate::crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}; +use crate::crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive, CryptoTypeId}; #[cfg(feature = "full_crypto")] use secp256k1::{PublicKey, SecretKey}; +/// An identifier used to match public keys against ecdsa keys +pub const CRYPTO_ID: CryptoTypeId = CryptoTypeId(*b"ecds"); + /// A secret seed (which is bytewise essentially equivalent to a SecretKey). /// /// We need it as a different type because `Seed` is expected to be AsRef<[u8]>. diff --git a/primitives/core/src/ed25519.rs b/primitives/core/src/ed25519.rs index 5e04dcceffb395fe5433bfcb28a217f8c44f06b5..abeac05388d2773e7ba80f60d6b4e9193ebe2fcd 100644 --- a/primitives/core/src/ed25519.rs +++ b/primitives/core/src/ed25519.rs @@ -38,10 +38,13 @@ use crate::crypto::{Pair as TraitPair, DeriveJunction, SecretStringError}; use crate::crypto::Ss58Codec; #[cfg(feature = "std")] use serde::{de, Serializer, Serialize, Deserializer, Deserialize}; -use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}}; +use crate::crypto::{Public as TraitPublic, CryptoTypePublicPair, UncheckedFrom, CryptoType, Derive, CryptoTypeId}; use sp_runtime_interface::pass_by::PassByInner; use sp_std::ops::Deref; +/// An identifier used to match public keys against ed25519 keys +pub const CRYPTO_ID: CryptoTypeId = CryptoTypeId(*b"ed25"); + /// A secret seed. It's not called a "secret key" because ring doesn't expose the secret keys /// of the key pair (yeah, dumb); as such we're forced to remember the seed manually if we /// will need it later (such as for HDKD). @@ -378,6 +381,18 @@ impl TraitPublic for Public { impl Derive for Public {} +impl From for CryptoTypePublicPair { + fn from(key: Public) -> Self { + (&key).into() + } +} + +impl From<&Public> for CryptoTypePublicPair { + fn from(key: &Public) -> Self { + CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) + } +} + /// Derive a single hard junction. #[cfg(feature = "full_crypto")] fn derive_hard_junction(secret_seed: &Seed, cc: &[u8; 32]) -> Seed { diff --git a/primitives/core/src/sr25519.rs b/primitives/core/src/sr25519.rs index 3495f32872f1694abe6756b643acb3ab7b7d3891..717952eb01c71b7cb398c5c57efc040704e07d24 100644 --- a/primitives/core/src/sr25519.rs +++ b/primitives/core/src/sr25519.rs @@ -39,7 +39,7 @@ use crate::crypto::{ #[cfg(feature = "std")] use crate::crypto::Ss58Codec; -use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}}; +use crate::crypto::{Public as TraitPublic, CryptoTypePublicPair, UncheckedFrom, CryptoType, Derive, CryptoTypeId}; use crate::hash::{H256, H512}; use codec::{Encode, Decode}; use sp_std::ops::Deref; @@ -54,6 +54,9 @@ use sp_runtime_interface::pass_by::PassByInner; #[cfg(feature = "full_crypto")] const SIGNING_CTX: &[u8] = b"substrate"; +/// An identifier used to match public keys against sr25519 keys +pub const CRYPTO_ID: CryptoTypeId = CryptoTypeId(*b"sr25"); + /// An Schnorrkel/Ristretto x25519 ("sr25519") public key. #[cfg_attr(feature = "full_crypto", derive(Hash))] #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Default, PassByInner)] @@ -390,6 +393,18 @@ impl TraitPublic for Public { } } +impl From for CryptoTypePublicPair { + fn from(key: Public) -> Self { + (&key).into() + } +} + +impl From<&Public> for CryptoTypePublicPair { + fn from(key: &Public) -> Self { + CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) + } +} + #[cfg(feature = "std")] impl From for Pair { fn from(sec: MiniSecretKey) -> Pair { @@ -529,25 +544,24 @@ impl TraitPair for Pair { self.0.sign(context.bytes(message)).into() } - /// Verify a signature on a message. Returns true if the signature is good. fn verify>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool { Self::verify_weak(&sig.0[..], message, pubkey) } - /// Verify a signature on a message. Returns true if the signature is good. fn verify_weak, M: AsRef<[u8]>>(sig: &[u8], message: M, pubkey: P) -> bool { - // Match both schnorrkel 0.1.1 and 0.8.0+ signatures, supporting both wallets - // that have not been upgraded and those that have. To swap to 0.8.0 only, - // create `schnorrkel::Signature` and pass that into `verify_simple` - match PublicKey::from_bytes(pubkey.as_ref()) { - Ok(pk) => pk.verify_simple_preaudit_deprecated( - SIGNING_CTX, message.as_ref(), &sig, - ).is_ok(), - Err(_) => false, - } + let signature = match schnorrkel::Signature::from_bytes(sig) { + Ok(signature) => signature, + Err(_) => return false, + }; + + let pub_key = match PublicKey::from_bytes(pubkey.as_ref()) { + Ok(pub_key) => pub_key, + Err(_) => return false, + }; + + pub_key.verify_simple(SIGNING_CTX, message.as_ref(), &signature).is_ok() } - /// Return a vec filled with raw data. fn to_raw_vec(&self) -> Vec { self.0.secret.to_bytes().to_vec() } @@ -566,6 +580,20 @@ impl Pair { let kp = mini_key.expand_to_keypair(ExpansionMode::Ed25519); (Pair(kp), mini_key.to_bytes()) } + + /// Verify a signature on a message. Returns `true` if the signature is good. + /// Supports old 0.1.1 deprecated signatures and should be used only for backward + /// compatibility. + pub fn verify_deprecated>(sig: &Signature, message: M, pubkey: &Public) -> bool { + // Match both schnorrkel 0.1.1 and 0.8.0+ signatures, supporting both wallets + // that have not been upgraded and those that have. + match PublicKey::from_bytes(pubkey.as_ref()) { + Ok(pk) => pk.verify_simple_preaudit_deprecated( + SIGNING_CTX, message.as_ref(), &sig.0[..], + ).is_ok(), + Err(_) => false, + } + } } impl CryptoType for Public { @@ -586,7 +614,7 @@ impl CryptoType for Pair { #[cfg(test)] mod compatibility_test { use super::*; - use crate::crypto::{DEV_PHRASE}; + use crate::crypto::DEV_PHRASE; use hex_literal::hex; // NOTE: tests to ensure addresses that are created with the `0.1.x` version (pre-audit) are @@ -609,14 +637,15 @@ mod compatibility_test { } #[test] - fn verify_known_message_should_work() { + fn verify_known_old_message_should_work() { let public = Public::from_raw(hex!("b4bfa1f7a5166695eb75299fd1c4c03ea212871c342f2c5dfea0902b2c246918")); // signature generated by the 1.1 version with the same ^^ public key. let signature = Signature::from_raw(hex!( "5a9755f069939f45d96aaf125cf5ce7ba1db998686f87f2fb3cbdea922078741a73891ba265f70c31436e18a9acd14d189d73c12317ab6c313285cd938453202" )); let message = b"Verifying that I am the owner of 5G9hQLdsKQswNPgB499DeA5PkFBbgkLPJWkkS6FAM6xGQ8xD. Hash: 221455a3\n"; - assert!(Pair::verify(&signature, &message[..], &public)); + assert!(Pair::verify_deprecated(&signature, &message[..], &public)); + assert!(!Pair::verify(&signature, &message[..], &public)); } } @@ -776,7 +805,7 @@ mod test { } #[test] - fn verify_from_wasm_works() { + fn verify_from_old_wasm_works() { // The values in this test case are compared to the output of `node-test.js` in schnorrkel-js. // // This is to make sure that the wasm library is compatible. @@ -787,7 +816,8 @@ mod test { let js_signature = Signature::from_raw(hex!( "28a854d54903e056f89581c691c1f7d2ff39f8f896c9e9c22475e60902cc2b3547199e0e91fa32902028f2ca2355e8cdd16cfe19ba5e8b658c94aa80f3b81a00" )); - assert!(Pair::verify(&js_signature, b"SUBSTRATE", &public)); + assert!(Pair::verify_deprecated(&js_signature, b"SUBSTRATE", &public)); + assert!(!Pair::verify(&js_signature, b"SUBSTRATE", &public)); } #[test] diff --git a/primitives/core/src/testing.rs b/primitives/core/src/testing.rs index 247323f9703b41ecf5a22897da4e4b0c6de1fb12..b5e6f4c7aff37427573de40587b9bfe6bf311281 100644 --- a/primitives/core/src/testing.rs +++ b/primitives/core/src/testing.rs @@ -16,10 +16,16 @@ //! Types that should only be used for testing! +use crate::crypto::{KeyTypeId, CryptoTypePublicPair}; #[cfg(feature = "std")] -use crate::{ed25519, sr25519, crypto::{Public, Pair}}; -use crate::crypto::KeyTypeId; - +use crate::{ + crypto::{Pair, Public}, + ed25519, sr25519, + traits::BareCryptoStoreError +}; +#[cfg(feature = "std")] +use std::collections::HashSet; +use codec::Encode; /// Key type for generic Ed25519 key. pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25"); /// Key type for generic Sr 25519 key. @@ -39,10 +45,41 @@ impl KeyStore { pub fn new() -> crate::traits::BareCryptoStorePtr { std::sync::Arc::new(parking_lot::RwLock::new(Self::default())) } + + fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option { + self.keys.get(&id) + .and_then(|inner| + inner.get(pub_key.as_slice()) + .map(|s| sr25519::Pair::from_string(s, None).expect("`sr25519` seed slice is valid")) + ) + } + + fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option { + self.keys.get(&id) + .and_then(|inner| + inner.get(pub_key.as_slice()) + .map(|s| ed25519::Pair::from_string(s, None).expect("`ed25519` seed slice is valid")) + ) + } + } #[cfg(feature = "std")] impl crate::traits::BareCryptoStore for KeyStore { + fn keys(&self, id: KeyTypeId) -> Result, BareCryptoStoreError> { + self.keys + .get(&id) + .map(|map| { + Ok(map.keys() + .fold(Vec::new(), |mut v, k| { + v.push(CryptoTypePublicPair(sr25519::CRYPTO_ID, k.clone())); + v.push(CryptoTypePublicPair(ed25519::CRYPTO_ID, k.clone())); + v + })) + }) + .unwrap_or(Ok(vec![])) + } + fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec { self.keys.get(&id) .map(|keys| @@ -58,10 +95,11 @@ impl crate::traits::BareCryptoStore for KeyStore { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result { + ) -> Result { match seed { Some(seed) => { - let pair = sr25519::Pair::from_string(seed, None).expect("Generates an `sr25519` pair."); + let pair = sr25519::Pair::from_string(seed, None) + .map_err(|_| BareCryptoStoreError::ValidationError("Generates an `sr25519` pair.".to_owned()))?; self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into()); Ok(pair.public()) }, @@ -73,14 +111,6 @@ impl crate::traits::BareCryptoStore for KeyStore { } } - fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option { - self.keys.get(&id) - .and_then(|inner| - inner.get(pub_key.as_slice()) - .map(|s| sr25519::Pair::from_string(s, None).expect("`sr25519` seed slice is valid")) - ) - } - fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec { self.keys.get(&id) .map(|keys| @@ -96,10 +126,11 @@ impl crate::traits::BareCryptoStore for KeyStore { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result { + ) -> Result { match seed { Some(seed) => { - let pair = ed25519::Pair::from_string(seed, None).expect("Generates an `ed25519` pair."); + let pair = ed25519::Pair::from_string(seed, None) + .map_err(|_| BareCryptoStoreError::ValidationError("Generates an `ed25519` pair.".to_owned()))?; self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into()); Ok(pair.public()) }, @@ -111,14 +142,6 @@ impl crate::traits::BareCryptoStore for KeyStore { } } - fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option { - self.keys.get(&id) - .and_then(|inner| - inner.get(pub_key.as_slice()) - .map(|s| ed25519::Pair::from_string(s, None).expect("`ed25519` seed slice is valid")) - ) - } - fn insert_unknown(&mut self, id: KeyTypeId, suri: &str, public: &[u8]) -> Result<(), ()> { self.keys.entry(id).or_default().insert(public.to_owned(), suri.to_string()); Ok(()) @@ -131,6 +154,40 @@ impl crate::traits::BareCryptoStore for KeyStore { fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool { public_keys.iter().all(|(k, t)| self.keys.get(&t).and_then(|s| s.get(k)).is_some()) } + + fn supported_keys( + &self, + id: KeyTypeId, + keys: Vec, + ) -> std::result::Result, BareCryptoStoreError> { + let provided_keys = keys.into_iter().collect::>(); + let all_keys = self.keys(id)?.into_iter().collect::>(); + + Ok(provided_keys.intersection(&all_keys).cloned().collect()) + } + + fn sign_with( + &self, + id: KeyTypeId, + key: &CryptoTypePublicPair, + msg: &[u8], + ) -> Result, BareCryptoStoreError> { + match key.0 { + ed25519::CRYPTO_ID => { + let key_pair: ed25519::Pair = self + .ed25519_key_pair(id, &ed25519::Public::from_slice(key.1.as_slice())) + .ok_or(BareCryptoStoreError::PairNotFound("ed25519".to_owned()))?; + return Ok(key_pair.sign(msg).encode()); + } + sr25519::CRYPTO_ID => { + let key_pair: sr25519::Pair = self + .sr25519_key_pair(id, &sr25519::Public::from_slice(key.1.as_slice())) + .ok_or(BareCryptoStoreError::PairNotFound("sr25519".to_owned()))?; + return Ok(key_pair.sign(msg).encode()); + } + _ => Err(BareCryptoStoreError::KeyNotSupported(id)) + } + } } /// Macro for exporting functions from wasm in with the expected signature for using it with the @@ -247,11 +304,9 @@ mod tests { .ed25519_generate_new(ED25519, None) .expect("Generates key"); - let store_key_pair = store.read() - .ed25519_key_pair(ED25519, &public) - .expect("Key should exists in store"); + let public_keys = store.read().keys(ED25519).unwrap(); - assert_eq!(public, store_key_pair.public()); + assert!(public_keys.contains(&public.into())); } #[test] @@ -267,11 +322,8 @@ mod tests { key_pair.public().as_ref(), ).expect("Inserts unknown key"); - let store_key_pair = store.read().sr25519_key_pair( - SR25519, - &key_pair.public(), - ).expect("Gets key pair from keystore"); + let public_keys = store.read().keys(SR25519).unwrap(); - assert_eq!(key_pair.public(), store_key_pair.public()); + assert!(public_keys.contains(&key_pair.public().into())); } } diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index 21a0b77ca7ab4ca0d7345928bc03c2d2e569a674..14839fb58562a3ec1bcff7219e6dbfb7e1b47b4d 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -16,14 +16,35 @@ //! Shareable Substrate traits. -use crate::{crypto::KeyTypeId, ed25519, sr25519}; +use crate::{ + crypto::{KeyTypeId, CryptoTypePublicPair}, + ed25519, sr25519, +}; use std::{ - fmt::{Debug, Display}, panic::UnwindSafe, sync::Arc, borrow::Cow, + borrow::Cow, + fmt::{Debug, Display}, + panic::UnwindSafe, + sync::Arc, }; pub use sp_externalities::{Externalities, ExternalitiesExt}; +/// BareCryptoStore error +#[derive(Debug)] +pub enum BareCryptoStoreError { + /// Public key type is not supported + KeyNotSupported(KeyTypeId), + /// Pair not found for public key and KeyTypeId + PairNotFound(String), + /// Validation error + ValidationError(String), + /// Keystore unavailable + Unavailable, + /// Programming errors + Other(String) +} + /// Something that generates, stores and provides access to keys. pub trait BareCryptoStore: Send + Sync { /// Returns all sr25519 public keys for the given key type. @@ -37,10 +58,7 @@ pub trait BareCryptoStore: Send + Sync { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result; - /// Returns the sr25519 key pair for the given key type and public key combination. - fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option; - + ) -> Result; /// Returns all ed25519 public keys for the given key type. fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec; /// Generate a new ed25519 key pair for the given key type and an optional seed. @@ -52,10 +70,7 @@ pub trait BareCryptoStore: Send + Sync { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result; - - /// Returns the ed25519 key pair for the given key type and public key combination. - fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option; + ) -> Result; /// Insert a new key. This doesn't require any known of the crypto; but a public key must be /// manually provided. @@ -67,11 +82,78 @@ pub trait BareCryptoStore: Send + Sync { /// Get the password for this store. fn password(&self) -> Option<&str>; + /// Find intersection between provided keys and supported keys + /// + /// Provided a list of (CryptoTypeId,[u8]) pairs, this would return + /// a filtered set of public keys which are supported by the keystore. + fn supported_keys( + &self, + id: KeyTypeId, + keys: Vec + ) -> Result, BareCryptoStoreError>; + /// List all supported keys + /// + /// Returns a set of public keys the signer supports. + fn keys(&self, id: KeyTypeId) -> Result, BareCryptoStoreError>; /// Checks if the private keys for the given public key and key type combinations exist. /// /// Returns `true` iff all private keys could be found. fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool; + + /// Sign with key + /// + /// Signs a message with the private key that matches + /// the public key passed. + /// + /// Returns the SCALE encoded signature if key is found & supported, + /// an error otherwise. + fn sign_with( + &self, + id: KeyTypeId, + key: &CryptoTypePublicPair, + msg: &[u8], + ) -> Result, BareCryptoStoreError>; + + /// Sign with any key + /// + /// Given a list of public keys, find the first supported key and + /// sign the provided message with that key. + /// + /// Returns a tuple of the used key and the signature + fn sign_with_any( + &self, + id: KeyTypeId, + keys: Vec, + msg: &[u8] + ) -> Result<(CryptoTypePublicPair, Vec), BareCryptoStoreError> { + if keys.len() == 1 { + return self.sign_with(id, &keys[0], msg).map(|s| (keys[0].clone(), s)); + } else { + for k in self.supported_keys(id, keys)? { + if let Ok(sign) = self.sign_with(id, &k, msg) { + return Ok((k, sign)); + } + } + } + Err(BareCryptoStoreError::KeyNotSupported(id)) + } + + /// Sign with all keys + /// + /// Provided a list of public keys, sign a message with + /// each key given that the key is supported. + /// + /// Returns a list of `Result`s each representing the signature of each key or + /// a BareCryptoStoreError for non-supported keys. + fn sign_with_all( + &self, + id: KeyTypeId, + keys: Vec, + msg: &[u8], + ) -> Result, BareCryptoStoreError>>, ()>{ + Ok(keys.iter().map(|k| self.sign_with(id, k, msg)).collect()) + } } /// A pointer to the key store. diff --git a/primitives/debug-derive/Cargo.toml b/primitives/debug-derive/Cargo.toml index 83db220461bc0ad5ee4fb9a882a44460d6318b32..0079b6219f5b618c709c322cf5639b9f6eec85f9 100644 --- a/primitives/debug-derive/Cargo.toml +++ b/primitives/debug-derive/Cargo.toml @@ -22,3 +22,6 @@ std = [] [dev-dependencies] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 3eed98bd6e0ff78b5213adf86830aac8072e128b..af6e516fbfcd32eec31dd9d446785a5028a70ddd 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -13,3 +13,6 @@ documentation = "https://docs.rs/sp-externalities" sp-storage = { version = "2.0.0-alpha.5", path = "../storage" } sp-std = { version = "2.0.0-alpha.5", path = "../std" } environmental = { version = "1.1.1" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 6fbd239b89cef8995d2ab05d7192e710fcc87c34..6a7f94394710f3702df3c9d9e3404df667e7b99a 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -39,10 +39,14 @@ pub trait Externalities: ExtensionStore { /// Read runtime storage. fn storage(&self, key: &[u8]) -> Option>; - /// Get storage value hash. This may be optimized for large values. + /// Get storage value hash. + /// + /// This may be optimized for large values. fn storage_hash(&self, key: &[u8]) -> Option>; - /// Get child storage value hash. This may be optimized for large values. + /// Get child storage value hash. + /// + /// This may be optimized for large values. /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage_hash( @@ -136,7 +140,7 @@ pub trait Externalities: ExtensionStore { /// Set or clear a storage entry (`key`) of current contract being called (effective immediately). fn place_storage(&mut self, key: Vec, value: Option>); - /// Set or clear a child storage entry. Return whether the operation succeeds. + /// Set or clear a child storage entry. fn place_child_storage( &mut self, storage_key: ChildStorageKey, @@ -148,16 +152,17 @@ pub trait Externalities: ExtensionStore { /// Get the identity of the chain. fn chain_id(&self) -> u64; - /// Get the trie root of the current storage map. This will also update all child storage keys - /// in the top-level storage map. + /// Get the trie root of the current storage map. /// - /// The hash is defined by the `Block`. + /// This will also update all child storage keys in the top-level storage map. /// - /// Returns the SCALE encoded hash. + /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_root(&mut self) -> Vec; - /// Get the trie root of a child storage map. This will also update the value of the child - /// storage keys in the top-level storage map. + /// Get the trie root of a child storage map. + /// + /// This will also update the value of the child storage keys in the top-level storage map. + /// /// If the storage root equals the default hash as defined by the trie, the key in the top-level /// storage map will be removed. fn child_storage_root( @@ -165,12 +170,11 @@ pub trait Externalities: ExtensionStore { storage_key: ChildStorageKey, ) -> Vec; - /// Get the change trie root of the current storage overlay at a block with given parent. - /// `parent` is expects a SCALE encoded hash. + /// Get the changes trie root of the current storage overlay at a block with given `parent`. /// - /// The hash is defined by the `Block`. + /// `parent` expects a SCALE encoded hash. /// - /// Returns the SCALE encoded hash. + /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/primitives/finality-grandpa/Cargo.toml b/primitives/finality-grandpa/Cargo.toml index 94bcd2bce42f2e8a648e78ab3fcdc7b84ac7ab5f..0595fa7ba700649fa2c4df5ae1ad3178648fdc1f 100644 --- a/primitives/finality-grandpa/Cargo.toml +++ b/primitives/finality-grandpa/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/sp-finality-grandpa" [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } @@ -28,3 +28,6 @@ std = [ "sp-api/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/finality-tracker/Cargo.toml b/primitives/finality-tracker/Cargo.toml index e6cf53495a7aa3c6c6e6ec9f7a152c8d44c9fbe3..4e6cf6c92d324c36278c18ef0b5b5703b21b56e3 100644 --- a/primitives/finality-tracker/Cargo.toml +++ b/primitives/finality-tracker/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME module that tracks the last finalized block, as perceived by block authors." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } @@ -20,3 +20,6 @@ std = [ "sp-std/std", "sp-inherents/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/inherents/Cargo.toml b/primitives/inherents/Cargo.toml index 8cc9cb4dd07ccfcfaf24f64c0220c3c61edd67a6..dd640f00ec1a272d87511bd2f22f88916abeebbb 100644 --- a/primitives/inherents/Cargo.toml +++ b/primitives/inherents/Cargo.toml @@ -14,7 +14,7 @@ documentation = "https://docs.rs/sp-inherents" parking_lot = { version = "0.10.0", optional = true } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } derive_more = { version = "0.99.2", optional = true } [features] @@ -26,3 +26,6 @@ std = [ "sp-core/std", "derive_more", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 7531df7e81a72e6a8b0ebc9a8a14f1c490248ce3..9cda2120cc2b45290da425f14f758ba483445b43 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/sp-io" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } hash-db = { version = "0.15.2", default-features = false } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } @@ -46,3 +46,6 @@ std = [ disable_panic_handler = [] disable_oom = [] disable_allocator = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 2702be02071af0def98a5aae5f45e1583df7583a..bc49df159eb6f873d5a9ac7e011a529cc2422cc0 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -286,7 +286,8 @@ pub trait Storage { /// /// The hashing algorithm is defined by the `Block`. /// - /// Returns an `Option` that holds the SCALE encoded hash. + /// Returns an `Some(_)` which holds the SCALE encoded hash or `None` when + /// changes trie is disabled. fn changes_root(&mut self, parent_hash: &[u8]) -> Option> { self.storage_changes_root(parent_hash) .expect("Invalid `parent_hash` given to `changes_root`.") @@ -410,8 +411,9 @@ pub trait Crypto { self.extension::() .expect("No `keystore` associated for the current context!") .read() - .ed25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) + .sign_with(id, &pub_key.into(), msg) + .map(|sig| ed25519::Signature::from_slice(sig.as_slice())) + .ok() } /// Verify an `ed25519` signature. @@ -462,13 +464,23 @@ pub trait Crypto { self.extension::() .expect("No `keystore` associated for the current context!") .read() - .sr25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) + .sign_with(id, &pub_key.into(), msg) + .map(|sig| sr25519::Signature::from_slice(sig.as_slice())) + .ok() + } + + /// Verify an `sr25519` signature. + /// + /// Returns `true` when the verification in successful regardless of + /// signature version. + fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { + sr25519::Pair::verify_deprecated(sig, msg, pubkey) } /// Verify an `sr25519` signature. /// /// Returns `true` when the verification in successful. + #[version(2)] fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { sr25519::Pair::verify(sig, msg, pubkey) } diff --git a/primitives/keyring/Cargo.toml b/primitives/keyring/Cargo.toml index 29c0a35a545e3c817761e29ad38a11e9f1c5856b..0764146250446c367c0c0bfbd53147e0a9dd4134 100644 --- a/primitives/keyring/Cargo.toml +++ b/primitives/keyring/Cargo.toml @@ -15,3 +15,6 @@ sp-core = { version = "2.0.0-alpha.5", path = "../core" } sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } lazy_static = "1.4.0" strum = { version = "0.16.0", features = ["derive"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/offchain/Cargo.toml b/primitives/offchain/Cargo.toml index a92f7e20c4d8bad0f99f68a8da7bbd8feccb86ea..66febccd59b9b6ac5a9111cfbd4fcb8027e8113a 100644 --- a/primitives/offchain/Cargo.toml +++ b/primitives/offchain/Cargo.toml @@ -18,3 +18,6 @@ std = [ "sp-api/std", "sp-runtime/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/panic-handler/Cargo.toml b/primitives/panic-handler/Cargo.toml index e16cc5dc7606c3865ef7da8db1ad5ec375705a21..169443f6c42cee9f40de81baeb04fe5d49a8ba76 100644 --- a/primitives/panic-handler/Cargo.toml +++ b/primitives/panic-handler/Cargo.toml @@ -12,3 +12,6 @@ documentation = "https://docs.rs/sp-panic-handler" [dependencies] backtrace = "0.3.38" log = "0.4.8" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/Cargo.toml b/primitives/phragmen/Cargo.toml index 2290b44c5ff0bb801441d94a6b7bf8f0746c1aa9..f5d26e8a40b80427360bfd56915a540d84d584bc 100644 --- a/primitives/phragmen/Cargo.toml +++ b/primitives/phragmen/Cargo.toml @@ -6,22 +6,29 @@ edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "PHRAGMENT primitives" +description = "Phragmen primitives" [dependencies] +codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } +sp-phragmen-compact = { version = "2.0.0-alpha.4", path = "./compact" } [dev-dependencies] substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } -sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } -rand = "0.7.2" +rand = "0.7.3" +sp-phragmen = { path = "." } [features] default = ["std"] +bench = [] std = [ + "codec/std", "serde", "sp-std/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/benches/phragmen.rs b/primitives/phragmen/benches/phragmen.rs index aa99e6f384ba65fd1d4621d3a3e608cb632e1a12..33da0b19563a8d3f7ef226077862558b5e0fd285 100644 --- a/primitives/phragmen/benches/phragmen.rs +++ b/primitives/phragmen/benches/phragmen.rs @@ -16,6 +16,7 @@ //! Note that execution times will not be accurate in an absolute scale, since //! - Everything is executed in the context of `TestExternalities` //! - Everything is executed in native environment. + #![cfg(feature = "bench")] #![feature(test)] diff --git a/primitives/phragmen/compact/Cargo.toml b/primitives/phragmen/compact/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..56b4520c54277825b7dc340b8fef7a9f132ca43a --- /dev/null +++ b/primitives/phragmen/compact/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sp-phragmen-compact" +version = "2.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Phragmen Compact Solution" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.7", features = ["full", "visit"] } +quote = "1.0" +proc-macro2 = "1.0.6" +proc-macro-crate = "0.1.4" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/compact/src/assignment.rs b/primitives/phragmen/compact/src/assignment.rs new file mode 100644 index 0000000000000000000000000000000000000000..587e482ccb22096182404dd04823c05749d4575b --- /dev/null +++ b/primitives/phragmen/compact/src/assignment.rs @@ -0,0 +1,210 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Code generation for the ratio assignment type. + +use crate::field_name_for; +use proc_macro2::{TokenStream as TokenStream2}; +use syn::{GenericArgument}; +use quote::quote; + +fn from_impl(count: usize) -> TokenStream2 { + let from_impl_single = { + let name = field_name_for(1); + quote!(1 => compact.#name.push( + ( + index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, + index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ) + ),) + }; + + let from_impl_double = { + let name = field_name_for(2); + quote!(2 => compact.#name.push( + ( + index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ( + index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + distribution[0].1, + ), + index_of_target(&distribution[1].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ) + ),) + }; + + let from_impl_rest = (3..=count).map(|c| { + let inner = (0..c-1).map(|i| + quote!((index_of_target(&distribution[#i].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, distribution[#i].1),) + ).collect::(); + + let field_name = field_name_for(c); + let last_index = c - 1; + let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?); + + quote!( + #c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)), + ) + }).collect::(); + + quote!( + #from_impl_single + #from_impl_double + #from_impl_rest + ) +} + +fn into_impl(count: usize) -> TokenStream2 { + let into_impl_single = { + let name = field_name_for(1); + quote!( + for (voter_index, target_index) in self.#name { + assignments.push(_phragmen::Assignment { + who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, + distribution: vec![ + (target_at(target_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, Accuracy::one()) + ], + }) + } + ) + }; + + let into_impl_double = { + let name = field_name_for(2); + quote!( + for (voter_index, (t1_idx, p1), t2_idx) in self.#name { + if p1 >= Accuracy::one() { + return Err(_phragmen::Error::CompactStakeOverflow); + } + + // defensive only. Since Percent doesn't have `Sub`. + let p2 = _phragmen::sp_runtime::traits::Saturating::saturating_sub( + Accuracy::one(), + p1, + ); + + assignments.push( _phragmen::Assignment { + who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, + distribution: vec![ + (target_at(t1_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p1), + (target_at(t2_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p2), + ] + }); + } + ) + }; + + let into_impl_rest = (3..=count).map(|c| { + let name = field_name_for(c); + quote!( + for (voter_index, inners, t_last_idx) in self.#name { + let mut sum = Accuracy::zero(); + let mut inners_parsed = inners + .iter() + .map(|(ref t_idx, p)| { + sum = _phragmen::sp_runtime::traits::Saturating::saturating_add(sum, *p); + let target = target_at(*t_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?; + Ok((target, *p)) + }) + .collect::, _phragmen::Error>>()?; + + if sum >= Accuracy::one() { + return Err(_phragmen::Error::CompactStakeOverflow); + } + + // defensive only. Since Percent doesn't have `Sub`. + let p_last = _phragmen::sp_runtime::traits::Saturating::saturating_sub( + Accuracy::one(), + sum, + ); + + inners_parsed.push((target_at(t_last_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, p_last)); + + assignments.push(_phragmen::Assignment { + who: voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, + distribution: inners_parsed, + }); + } + ) + }).collect::(); + + quote!( + #into_impl_single + #into_impl_double + #into_impl_rest + ) +} + +pub(crate) fn assignment( + ident: syn::Ident, + voter_type: GenericArgument, + target_type: GenericArgument, + count: usize, +) -> TokenStream2 { + + let from_impl = from_impl(count); + let into_impl = into_impl(count); + + quote!( + impl< + #voter_type: _phragmen::codec::Codec + Default + Copy, + #target_type: _phragmen::codec::Codec + Default + Copy, + Accuracy: + _phragmen::codec::Codec + Default + Clone + _phragmen::sp_runtime::PerThing + + PartialOrd, + > + #ident<#voter_type, #target_type, Accuracy> + { + pub fn from_assignment( + assignments: Vec<_phragmen::Assignment>, + index_of_voter: FV, + index_of_target: FT, + ) -> Result + where + for<'r> FV: Fn(&'r A) -> Option<#voter_type>, + for<'r> FT: Fn(&'r A) -> Option<#target_type>, + A: _phragmen::IdentifierT, + { + let mut compact: #ident< + #voter_type, + #target_type, + Accuracy, + > = Default::default(); + + for _phragmen::Assignment { who, distribution } in assignments { + match distribution.len() { + 0 => continue, + #from_impl + _ => { + return Err(_phragmen::Error::CompactTargetOverflow); + } + } + }; + Ok(compact) + } + + pub fn into_assignment( + self, + voter_at: impl Fn(#voter_type) -> Option, + target_at: impl Fn(#target_type) -> Option, + ) -> Result>, _phragmen::Error> { + let mut assignments: Vec<_phragmen::Assignment> = Default::default(); + #into_impl + Ok(assignments) + } + } + ) +} diff --git a/primitives/phragmen/compact/src/lib.rs b/primitives/phragmen/compact/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..114aeaeb32ed92ae4badbb59d38dc77cbcf309e7 --- /dev/null +++ b/primitives/phragmen/compact/src/lib.rs @@ -0,0 +1,219 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Proc macro for phragmen compact assignment. + +use proc_macro::TokenStream; +use proc_macro2::{TokenStream as TokenStream2, Span, Ident}; +use proc_macro_crate::crate_name; +use quote::quote; +use syn::{GenericArgument, Type, parse::{Parse, ParseStream, Result}}; + +mod assignment; +mod staked; + +// prefix used for struct fields in compact. +const PREFIX: &'static str = "votes"; + +/// Generates a struct to store the phragmen assignments in a compact way. The struct can only store +/// distributions up to the given input count. The given count must be greater than 2. +/// +/// ```ignore +/// // generate a struct with nominator and edge weight u128, with maximum supported +/// // edge per voter of 16. +/// generate_compact_solution_type(pub TestCompact, 16) +/// ``` +/// +/// This generates: +/// +/// ```ignore +/// pub struct TestCompact { +/// votes1: Vec<(V, T)>, +/// votes2: Vec<(V, (T, W), T)>, +/// votes3: Vec<(V, [(T, W); 2usize], T)>, +/// votes4: Vec<(V, [(T, W); 3usize], T)>, +/// votes5: Vec<(V, [(T, W); 4usize], T)>, +/// votes6: Vec<(V, [(T, W); 5usize], T)>, +/// votes7: Vec<(V, [(T, W); 6usize], T)>, +/// votes8: Vec<(V, [(T, W); 7usize], T)>, +/// votes9: Vec<(V, [(T, W); 8usize], T)>, +/// votes10: Vec<(V, [(T, W); 9usize], T)>, +/// votes11: Vec<(V, [(T, W); 10usize], T)>, +/// votes12: Vec<(V, [(T, W); 11usize], T)>, +/// votes13: Vec<(V, [(T, W); 12usize], T)>, +/// votes14: Vec<(V, [(T, W); 13usize], T)>, +/// votes15: Vec<(V, [(T, W); 14usize], T)>, +/// votes16: Vec<(V, [(T, W); 15usize], T)>, +/// } +/// ``` +/// +/// The generic arguments are: +/// - `V`: identifier/index for voter (nominator) types. +/// - `T` identifier/index for candidate (validator) types. +/// - `W` weight type. +/// +/// Some conversion implementations are provided by default if +/// - `W` is u128, or +/// - `W` is anything that implements `PerThing` (such as `Perbill`) +/// +/// The ideas behind the structure are as follows: +/// +/// - For single distribution, no weight is stored. The weight is known to be 100%. +/// - For all the rest, the weight if the last distribution is omitted. This value can be computed +/// from the rest. +/// +#[proc_macro] +pub fn generate_compact_solution_type(item: TokenStream) -> TokenStream { + let CompactSolutionDef { + vis, + ident, + count, + } = syn::parse_macro_input!(item as CompactSolutionDef); + + let voter_type = GenericArgument::Type(Type::Verbatim(quote!(V))); + let target_type = GenericArgument::Type(Type::Verbatim(quote!(T))); + let weight_type = GenericArgument::Type(Type::Verbatim(quote!(W))); + + let imports = imports().unwrap_or_else(|e| e.to_compile_error()); + + let compact_def = struct_def( + vis, + ident.clone(), + count, + voter_type.clone(), + target_type.clone(), + weight_type, + ).unwrap_or_else(|e| e.to_compile_error()); + + let assignment_impls = assignment::assignment( + ident.clone(), + voter_type.clone(), + target_type.clone(), + count, + ); + + let staked_impls = staked::staked( + ident, + voter_type, + target_type, + count, + ); + + quote!( + #imports + #compact_def + #assignment_impls + #staked_impls + ).into() +} + +fn struct_def( + vis: syn::Visibility, + ident: syn::Ident, + count: usize, + voter_type: GenericArgument, + target_type: GenericArgument, + weight_type: GenericArgument, +) -> Result { + if count <= 2 { + Err(syn::Error::new( + Span::call_site(), + "cannot build compact solution struct with capacity less than 2." + ))? + } + + let singles = { + let name = field_name_for(1); + quote!(#name: Vec<(#voter_type, #target_type)>,) + }; + + let doubles = { + let name = field_name_for(2); + quote!(#name: Vec<(#voter_type, (#target_type, #weight_type), #target_type)>,) + }; + + let rest = (3..=count).map(|c| { + let field_name = field_name_for(c); + let array_len = c - 1; + quote!( + #field_name: Vec<( + #voter_type, + [(#target_type, #weight_type); #array_len], + #target_type + )>, + ) + }).collect::(); + + Ok(quote! ( + /// A struct to encode a Phragmen assignment in a compact way. + #[derive( + Default, + PartialEq, + Eq, + Clone, + _phragmen::sp_runtime::RuntimeDebug, + _phragmen::codec::Encode, + _phragmen::codec::Decode, + )] + #vis struct #ident<#voter_type, #target_type, #weight_type> { + // _marker: sp_std::marker::PhantomData, + #singles + #doubles + #rest + } + + impl<#voter_type, #target_type, #weight_type> _phragmen::VotingLimit + for #ident<#voter_type, #target_type, #weight_type> + { + const LIMIT: usize = #count; + } + )) +} + +fn imports() -> Result { + let sp_phragmen_imports = match crate_name("sp-phragmen") { + Ok(sp_phragmen) => { + let ident = syn::Ident::new(&sp_phragmen, Span::call_site()); + quote!( extern crate #ident as _phragmen; ) + } + Err(e) => return Err(syn::Error::new(Span::call_site(), &e)), + }; + + Ok(quote!( + #sp_phragmen_imports + )) +} + +struct CompactSolutionDef { + vis: syn::Visibility, + ident: syn::Ident, + count: usize, +} + +impl Parse for CompactSolutionDef { + fn parse(input: ParseStream) -> syn::Result { + let vis: syn::Visibility = input.parse()?; + let ident: syn::Ident = input.parse()?; + let _ = ::parse(input)?; + let count_literal: syn::LitInt = input.parse()?; + let count = count_literal.base10_parse::()?; + Ok(Self { vis, ident, count } ) + } +} + +fn field_name_for(n: usize) -> Ident { + Ident::new(&format!("{}{}", PREFIX, n), Span::call_site()) +} diff --git a/primitives/phragmen/compact/src/staked.rs b/primitives/phragmen/compact/src/staked.rs new file mode 100644 index 0000000000000000000000000000000000000000..a7cf853f17086651ef4be37dcf2538cdce936dc6 --- /dev/null +++ b/primitives/phragmen/compact/src/staked.rs @@ -0,0 +1,208 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Code generation for the staked assignment type. + +use crate::field_name_for; +use proc_macro2::{TokenStream as TokenStream2}; +use syn::{GenericArgument}; +use quote::quote; + +fn from_impl(count: usize) -> TokenStream2 { + let from_impl_single = { + let name = field_name_for(1); + quote!(1 => compact.#name.push( + ( + index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, + index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ) + ),) + }; + + let from_impl_double = { + let name = field_name_for(2); + quote!(2 => compact.#name.push( + ( + index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ( + index_of_target(&distribution[0].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + distribution[0].1, + ), + index_of_target(&distribution[1].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, + ) + ),) + }; + + let from_impl_rest = (3..=count).map(|c| { + let inner = (0..c-1).map(|i| + quote!((index_of_target(&distribution[#i].0).ok_or(_phragmen::Error::CompactInvalidIndex)?, distribution[#i].1),) + ).collect::(); + + let field_name = field_name_for(c); + let last_index = c - 1; + let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?); + + quote!( + #c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)), + ) + }).collect::(); + + quote!( + #from_impl_single + #from_impl_double + #from_impl_rest + ) +} + +fn into_impl(count: usize) -> TokenStream2 { + let into_impl_single = { + let name = field_name_for(1); + quote!( + for (voter_index, target_index) in self.#name { + let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; + let all_stake = max_of(&who); + assignments.push(_phragmen::StakedAssignment { + who, + distribution: vec![(target_at(target_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, all_stake)], + }) + } + ) + }; + + let into_impl_double = { + let name = field_name_for(2); + quote!( + for (voter_index, (t1_idx, w1), t2_idx) in self.#name { + let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; + let all_stake = max_of(&who); + + if w1 >= all_stake { + return Err(_phragmen::Error::CompactStakeOverflow); + } + + // w2 is ensured to be positive. + let w2 = all_stake - w1; + assignments.push( _phragmen::StakedAssignment { + who, + distribution: vec![ + (target_at(t1_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w1), + (target_at(t2_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w2), + ] + }); + } + ) + }; + + let into_impl_rest = (3..=count).map(|c| { + let name = field_name_for(c); + quote!( + for (voter_index, inners, t_last_idx) in self.#name { + let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; + let mut sum = u128::min_value(); + let all_stake = max_of(&who); + + let mut inners_parsed = inners + .iter() + .map(|(ref t_idx, w)| { + sum = sum.saturating_add(*w); + let target = target_at(*t_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?; + Ok((target, *w)) + }).collect::, _phragmen::Error>>()?; + + if sum >= all_stake { + return Err(_phragmen::Error::CompactStakeOverflow); + } + // w_last is proved to be positive. + let w_last = all_stake - sum; + + inners_parsed.push((target_at(t_last_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?, w_last)); + + assignments.push(_phragmen::StakedAssignment { + who, + distribution: inners_parsed, + }); + } + ) + }).collect::(); + + quote!( + #into_impl_single + #into_impl_double + #into_impl_rest + ) +} + +pub(crate) fn staked( + ident: syn::Ident, + voter_type: GenericArgument, + target_type: GenericArgument, + count: usize, +) -> TokenStream2 { + + let from_impl = from_impl(count); + let into_impl = into_impl(count); + quote!( + impl< + #voter_type: _phragmen::codec::Codec + Default + Copy, + #target_type: _phragmen::codec::Codec + Default + Copy, + > + #ident<#voter_type, #target_type, u128> + { + /// Generate self from a vector of `StakedAssignment`. + pub fn from_staked( + assignments: Vec<_phragmen::StakedAssignment>, + index_of_voter: FV, + index_of_target: FT, + ) -> Result + where + for<'r> FV: Fn(&'r A) -> Option<#voter_type>, + for<'r> FT: Fn(&'r A) -> Option<#target_type>, + A: _phragmen::IdentifierT + { + let mut compact: #ident<#voter_type, #target_type, u128> = Default::default(); + for _phragmen::StakedAssignment { who, distribution } in assignments { + match distribution.len() { + 0 => continue, + #from_impl + _ => { + return Err(_phragmen::Error::CompactTargetOverflow); + } + } + }; + Ok(compact) + } + + /// Convert self into `StakedAssignment`. The given function should return the total + /// weight of a voter. It is used to subtract the sum of all the encoded weights to + /// infer the last one. + pub fn into_staked( + self, + max_of: FM, + voter_at: impl Fn(#voter_type) -> Option, + target_at: impl Fn(#target_type) -> Option, + ) + -> Result>, _phragmen::Error> + where + for<'r> FM: Fn(&'r A) -> u128, + A: _phragmen::IdentifierT, + { + let mut assignments: Vec<_phragmen::StakedAssignment> = Default::default(); + #into_impl + Ok(assignments) + } + } + ) +} diff --git a/primitives/arithmetic/fuzzer/.gitignore b/primitives/phragmen/fuzzer/.gitignore similarity index 100% rename from primitives/arithmetic/fuzzer/.gitignore rename to primitives/phragmen/fuzzer/.gitignore diff --git a/primitives/phragmen/fuzzer/Cargo.lock b/primitives/phragmen/fuzzer/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..3ef2a2732424a3ddd412192dc01ac0647c36ffe1 --- /dev/null +++ b/primitives/phragmen/fuzzer/Cargo.lock @@ -0,0 +1,1602 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "ahash" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" +dependencies = [ + "const-random", +] + +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +dependencies = [ + "memchr", +] + +[[package]] +name = "arbitrary" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca797db0057bae1a7aa2eef3283a874695455cecf08a43bfb8507ee0ebc1ed69" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "base58" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +dependencies = [ + "arrayvec 0.4.12", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "byte-slice-cast" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clear_on_drop" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" +dependencies = [ + "cc", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "const-random" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" +dependencies = [ + "getrandom", + "proc-macro-hack", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle 1.0.0", +] + +[[package]] +name = "curve25519-dalek" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" +dependencies = [ + "byteorder", + "clear_on_drop", + "digest", + "rand_core 0.3.1", + "subtle 2.2.2", +] + +[[package]] +name = "curve25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" +dependencies = [ + "byteorder", + "digest", + "rand_core 0.5.1", + "subtle 2.2.2", + "zeroize 1.1.0", +] + +[[package]] +name = "derive_more" +version = "0.99.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.0-pre.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" +dependencies = [ + "clear_on_drop", + "curve25519-dalek 2.0.0", + "rand 0.7.3", + "sha2", +] + +[[package]] +name = "environmental" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" + +[[package]] +name = "failure" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fixed-hash" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" +dependencies = [ + "byteorder", + "libc", + "rand 0.7.3", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +dependencies = [ + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" +dependencies = [ + "ahash", + "autocfg 0.1.7", +] + +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "hmac-drbg" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" +dependencies = [ + "digest", + "generic-array", + "hmac", +] + +[[package]] +name = "honggfuzz" +version = "0.5.45" +dependencies = [ + "arbitrary", + "lazy_static", + "memmap", +] + +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-serde" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-serde" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bbe9ea9b182f0fb1cabbd61f4ff9b7b7b9197955e95a7e4c27de5055eb29ff8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "integer-sqrt" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65877bf7d44897a473350b1046277941cee20b263397e90869c50b6e766088b" + +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" + +[[package]] +name = "libsecp256k1" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" +dependencies = [ + "arrayref", + "crunchy", + "digest", + "hmac-drbg", + "rand 0.7.3", + "sha2", + "subtle 2.2.2", + "typenum", +] + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "memory-db" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" +dependencies = [ + "ahash", + "hash-db", + "hashbrown", + "parity-util-mem", +] + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + +[[package]] +name = "merlin" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" +dependencies = [ + "byteorder", + "keccak", + "rand_core 0.4.2", + "zeroize 1.1.0", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.0.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +dependencies = [ + "autocfg 1.0.0", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +dependencies = [ + "autocfg 1.0.0", +] + +[[package]] +name = "once_cell" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +dependencies = [ + "parking_lot 0.9.0", +] + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "parity-scale-codec" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" +dependencies = [ + "arrayvec 0.5.1", + "bitvec", + "byte-slice-cast", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parity-util-mem" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" +dependencies = [ + "cfg-if", + "impl-trait-for-tuples", + "parity-util-mem-derive", + "parking_lot 0.10.0", + "primitive-types", + "winapi", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +dependencies = [ + "lock_api", + "parking_lot_core 0.7.0", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec 1.3.0", + "winapi", +] + +[[package]] +name = "paste" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +dependencies = [ + "byteorder", + "crypto-mac", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" + +[[package]] +name = "primitive-types" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-serde 0.3.0", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "regex" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "schnorrkel" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" +dependencies = [ + "curve25519-dalek 1.2.3", + "failure", + "merlin", + "rand 0.6.5", + "rand_core 0.4.2", + "rand_os", + "sha2", + "subtle 2.2.2", + "zeroize 0.9.3", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" + +[[package]] +name = "sp-application-crypto" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-std", +] + +[[package]] +name = "sp-arithmetic" +version = "2.0.0-alpha.3" +dependencies = [ + "integer-sqrt", + "num-traits", + "parity-scale-codec", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-core" +version = "2.0.0-alpha.3" +dependencies = [ + "base58", + "blake2-rfc", + "byteorder", + "ed25519-dalek", + "hash-db", + "hash256-std-hasher", + "hex", + "impl-serde 0.3.0", + "lazy_static", + "libsecp256k1", + "log", + "num-traits", + "parity-scale-codec", + "parity-util-mem", + "parking_lot 0.10.0", + "primitive-types", + "rand 0.7.3", + "regex", + "rustc-hex", + "schnorrkel", + "serde", + "sha2", + "sp-debug-derive", + "sp-externalities", + "sp-runtime-interface", + "sp-std", + "sp-storage", + "substrate-bip39", + "tiny-bip39", + "tiny-keccak", + "twox-hash", + "wasmi", + "zeroize 1.1.0", +] + +[[package]] +name = "sp-debug-derive" +version = "2.0.0-alpha.3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-externalities" +version = "0.8.0-alpha.3" +dependencies = [ + "environmental", + "sp-std", + "sp-storage", +] + +[[package]] +name = "sp-inherents" +version = "2.0.0-alpha.3" +dependencies = [ + "derive_more", + "parity-scale-codec", + "parking_lot 0.10.0", + "sp-core", + "sp-std", +] + +[[package]] +name = "sp-io" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "libsecp256k1", + "log", + "parity-scale-codec", + "sp-core", + "sp-externalities", + "sp-runtime-interface", + "sp-state-machine", + "sp-std", + "sp-trie", + "sp-wasm-interface", +] + +[[package]] +name = "sp-panic-handler" +version = "2.0.0-alpha.3" +dependencies = [ + "backtrace", + "log", +] + +[[package]] +name = "sp-phragmen" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "serde", + "sp-core", + "sp-phragmen-compact", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-phragmen-compact" +version = "2.0.0-dev" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-phragmen-fuzzer" +version = "2.0.0" +dependencies = [ + "honggfuzz", + "rand 0.7.3", + "sp-phragmen", +] + +[[package]] +name = "sp-runtime" +version = "2.0.0-alpha.3" +dependencies = [ + "hash256-std-hasher", + "impl-trait-for-tuples", + "log", + "parity-scale-codec", + "parity-util-mem", + "paste", + "rand 0.7.3", + "serde", + "sp-application-crypto", + "sp-arithmetic", + "sp-core", + "sp-inherents", + "sp-io", + "sp-std", +] + +[[package]] +name = "sp-runtime-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "sp-externalities", + "sp-runtime-interface-proc-macro", + "sp-std", + "sp-wasm-interface", + "static_assertions", +] + +[[package]] +name = "sp-runtime-interface-proc-macro" +version = "2.0.0-alpha.3" +dependencies = [ + "Inflector", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-state-machine" +version = "0.8.0-alpha.3" +dependencies = [ + "hash-db", + "log", + "num-traits", + "parity-scale-codec", + "parking_lot 0.10.0", + "rand 0.7.3", + "sp-core", + "sp-externalities", + "sp-panic-handler", + "sp-trie", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-std" +version = "2.0.0-alpha.3" + +[[package]] +name = "sp-storage" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-serde 0.2.3", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-trie" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "memory-db", + "parity-scale-codec", + "sp-core", + "sp-std", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-wasm-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-std", + "wasmi", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substrate-bip39" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" +dependencies = [ + "hmac", + "pbkdf2", + "schnorrkel", + "sha2", +] + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + +[[package]] +name = "subtle" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" + +[[package]] +name = "syn" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tiny-bip39" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +dependencies = [ + "failure", + "hmac", + "once_cell", + "pbkdf2", + "rand 0.7.3", + "rustc-hash", + "sha2", + "unicode-normalization", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" +dependencies = [ + "serde", +] + +[[package]] +name = "trie-db" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" +dependencies = [ + "hash-db", + "hashbrown", + "log", + "rustc-hex", + "smallvec 1.3.0", +] + +[[package]] +name = "trie-root" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +dependencies = [ + "hash-db", +] + +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" + +[[package]] +name = "uint" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" +dependencies = [ + "byteorder", + "crunchy", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +dependencies = [ + "smallvec 1.3.0", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasmi" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" +dependencies = [ + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zeroize" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/primitives/phragmen/fuzzer/Cargo.toml b/primitives/phragmen/fuzzer/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..90af69a707f75a84b417e2729abf6a245061bb41 --- /dev/null +++ b/primitives/phragmen/fuzzer/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sp-phragmen-fuzzer" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +sp-phragmen = { version = "2.0.0-alpha.3", path = ".." } +honggfuzz = "0.5" +rand = "0.7.3" + +[workspace] + +[[bin]] +name = "reduce" +path = "src/reduce.rs" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/fuzzer/src/reduce.rs b/primitives/phragmen/fuzzer/src/reduce.rs new file mode 100644 index 0000000000000000000000000000000000000000..4bf08590a149feafae1fc9ab33372c24440377de --- /dev/null +++ b/primitives/phragmen/fuzzer/src/reduce.rs @@ -0,0 +1,145 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Running +//! +//! Run with `cargo hfuzz run reduce`. `honggfuzz`. +//! +//! # Debugging a panic +//! +//! Once a panic is found, it can be debugged with +//! `cargo hfuzz run-debug reduce hfuzz_workspace/reduce/*.fuzz`. + +use honggfuzz::fuzz; +use sp_phragmen::{StakedAssignment, ExtendedBalance, build_support_map, reduce}; +use rand::{self, Rng}; + +type Balance = u128; +type AccountId = u64; + +/// Or any other token type. +const KSM: Balance = 1_000_000_000_000; + +fn main() { + loop { + fuzz!(|_data: _| { + let (assignments, winners) = generate_random_phragmen_assignment( + rr(100, 1000), + rr(100, 2000), + 8, + 8, + ); + reduce_and_compare(&assignments, &winners); + }); + } +} + +fn generate_random_phragmen_assignment( + voter_count: usize, + target_count: usize, + avg_edge_per_voter: usize, + edge_per_voter_var: usize, +) -> (Vec>, Vec) { + // random in range of (a, b) + let rr_128 = |a: u128, b: u128| -> u128 { rand::thread_rng().gen_range(a, b) }; + + // prefix to distinguish the voter and target account ranges. + let target_prefix = 1_000_000; + // let target_prefix = 1000; + assert!(voter_count < target_prefix); + + let mut assignments = Vec::with_capacity(voter_count as usize); + let mut winners: Vec = Vec::new(); + + let all_targets = (target_prefix..(target_prefix + target_count)) + .map(|a| a as AccountId) + .collect::>(); + + (1..=voter_count).for_each(|acc| { + let mut targets_to_chose_from = all_targets.clone(); + let targets_to_chose = if edge_per_voter_var > 0 { rr( + avg_edge_per_voter - edge_per_voter_var, + avg_edge_per_voter + edge_per_voter_var, + ) } else { avg_edge_per_voter }; + + let distribution = (0..targets_to_chose).map(|_| { + let target = targets_to_chose_from.remove(rr(0, targets_to_chose_from.len())); + if winners.iter().find(|w| **w == target).is_none() { + winners.push(target.clone()); + } + (target, rr_128(1 * KSM, 100 * KSM)) + }).collect::>(); + + assignments.push(StakedAssignment { + who: (acc as AccountId), + distribution, + }); + }); + + (assignments, winners) +} + +fn assert_assignments_equal( + winners: &Vec, + ass1: &Vec>, + ass2: &Vec>, +) { + + let (support_1, _) = build_support_map::(winners, ass1); + let (support_2, _) = build_support_map::(winners, ass2); + + for (who, support) in support_1.iter() { + assert_eq!(support.total, support_2.get(who).unwrap().total); + } +} + +fn reduce_and_compare( + assignment: &Vec>, + winners: &Vec, +) { + let mut altered_assignment = assignment.clone(); + let n = assignment.len() as u32; + let m = winners.len() as u32; + + let edges_before = assignment_len(&assignment); + let num_changed = reduce(&mut altered_assignment); + let edges_after = edges_before - num_changed; + + assert!( + edges_after <= m + n, + "reduce bound not satisfied. n = {}, m = {}, edges after reduce = {} (removed {})", + n, + m, + edges_after, + num_changed, + ); + + assert_assignments_equal( + winners, + &assignment, + &altered_assignment, + ); +} + +fn assignment_len(assignments: &[StakedAssignment]) -> u32 { + let mut counter = 0; + assignments.iter().for_each(|x| x.distribution.iter().for_each(|_| counter += 1)); + counter +} + +fn rr(a: usize, b: usize) -> usize { + rand::thread_rng().gen_range(a, b) +} diff --git a/primitives/phragmen/src/helpers.rs b/primitives/phragmen/src/helpers.rs new file mode 100644 index 0000000000000000000000000000000000000000..27f51b4a05fe2c2c22a02f8442efdd9d9f86124d --- /dev/null +++ b/primitives/phragmen/src/helpers.rs @@ -0,0 +1,94 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Helper methods for phragmen. + +use crate::{Assignment, ExtendedBalance, IdentifierT, StakedAssignment}; +use sp_runtime::PerThing; +use sp_std::prelude::*; + +/// Converts a vector of ratio assignments into ones with absolute budget value. +pub fn assignment_ratio_to_staked( + ratio: Vec>, + stake_of: FS, +) -> Vec> +where + for<'r> FS: Fn(&'r A) -> ExtendedBalance, + T: sp_std::ops::Mul, + ExtendedBalance: From<::Inner>, +{ + ratio + .into_iter() + .map(|a| { + let stake = stake_of(&a.who); + a.into_staked(stake, true) + }) + .collect() +} + +/// Converts a vector of staked assignments into ones with ratio values. +pub fn assignment_staked_to_ratio( + ratio: Vec>, +) -> Vec> +where + ExtendedBalance: From<::Inner>, +{ + ratio.into_iter().map(|a| a.into_assignment(true)).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ExtendedBalance; + use sp_runtime::Perbill; + + #[test] + fn into_staked_works() { + let ratio = vec![ + Assignment { + who: 1u32, + distribution: vec![ + (10u32, Perbill::from_fraction(0.5)), + (20, Perbill::from_fraction(0.5)), + ], + }, + Assignment { + who: 2u32, + distribution: vec![ + (10, Perbill::from_fraction(0.33)), + (20, Perbill::from_fraction(0.67)), + ], + }, + ]; + + let stake_of = |_: &u32| -> ExtendedBalance { 100u128 }; + let staked = assignment_ratio_to_staked(ratio, stake_of); + + assert_eq!( + staked, + vec![ + StakedAssignment { + who: 1u32, + distribution: vec![(10u32, 50), (20, 50),] + }, + StakedAssignment { + who: 2u32, + distribution: vec![(10u32, 33), (20, 67),] + } + ] + ); + } +} diff --git a/primitives/phragmen/src/lib.rs b/primitives/phragmen/src/lib.rs index aba714faeb19f259e3a0aed5ad2e9514161ec408..c0d94a71e1fc209f4db7085bd8fcba0262d66ecd 100644 --- a/primitives/phragmen/src/lib.rs +++ b/primitives/phragmen/src/lib.rs @@ -33,19 +33,60 @@ #![cfg_attr(not(feature = "std"), no_std)] -use sp_std::{prelude::*, collections::btree_map::BTreeMap, convert::TryFrom}; -use sp_runtime::{ - PerThing, Rational128, RuntimeDebug, - helpers_128bit::multiply_by_rational, -}; -use sp_runtime::traits::{ - Zero, Convert, Member, AtLeast32Bit, SaturatedConversion, Bounded, Saturating, -}; +use sp_std::{prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, convert::TryFrom}; +use sp_runtime::{helpers_128bit::multiply_by_rational, PerThing, Rational128, RuntimeDebug, SaturatedConversion}; +use sp_runtime::traits::{Zero, Convert, Member, AtLeast32Bit, Saturating, Bounded}; #[cfg(test)] mod mock; #[cfg(test)] mod tests; +#[cfg(feature = "std")] +use serde::{Serialize, Deserialize}; +#[cfg(feature = "std")] +use codec::{Encode, Decode}; + +mod node; +mod reduce; +mod helpers; + +// re-export reduce stuff. +pub use reduce::reduce; + +// re-export the helpers. +pub use helpers::*; + +// re-export the compact macro, with the dependencies of the macro. +#[doc(hidden)] +pub use codec; +#[doc(hidden)] +pub use sp_runtime; + +// re-export the compact solution type. +pub use sp_phragmen_compact::generate_compact_solution_type; + +/// A trait to limit the number of votes per voter. The generated compact type will implement this. +pub trait VotingLimit { + const LIMIT: usize; +} + +/// an aggregator trait for a generic type of a voter/target identifier. This usually maps to +/// substrate's account id. +pub trait IdentifierT: Clone + Eq + Default + Ord + Debug + codec::Codec {} + +impl IdentifierT for T {} + +/// The errors that might occur in the this crate and compact. +#[derive(Debug, Eq, PartialEq)] +pub enum Error { + /// While going from compact to staked, the stake of all the edges has gone above the + /// total and the last stake cannot be assigned. + CompactStakeOverflow, + /// The compact type has a voter who's number of targets is out of bound. + CompactTargetOverflow, + /// One of the index functions returned none. + CompactInvalidIndex, +} /// A type in which performing operations on balances and stakes of candidates and voters are safe. /// @@ -55,6 +96,9 @@ mod tests; /// Balance types converted to `ExtendedBalance` are referred to as `Votes`. pub type ExtendedBalance = u128; +/// The score of an assignment. This can be computed from the support map via [`evaluate_support`]. +pub type PhragmenScore = [ExtendedBalance; 3]; + /// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we /// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number /// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128; @@ -62,11 +106,11 @@ const DEN: u128 = u128::max_value(); /// A candidate entity for phragmen election. #[derive(Clone, Default, RuntimeDebug)] -pub struct Candidate { +struct Candidate { /// Identifier. - pub who: AccountId, + who: AccountId, /// Intermediary value used to sort candidates. - pub score: Rational128, + score: Rational128, /// Sum of the stake of this candidate based on received votes. approval_stake: ExtendedBalance, /// Flag for being elected. @@ -75,7 +119,7 @@ pub struct Candidate { /// A voter entity. #[derive(Clone, Default, RuntimeDebug)] -pub struct Voter { +struct Voter { /// Identifier. who: AccountId, /// List of candidates proposed by this voter. @@ -88,7 +132,7 @@ pub struct Voter { /// A candidate being backed by a voter. #[derive(Clone, Default, RuntimeDebug)] -pub struct Edge { +struct Edge { /// Identifier. who: AccountId, /// Load of this vote. @@ -97,12 +141,6 @@ pub struct Edge { candidate_index: usize, } -/// Particular `AccountId` was backed by `T`-ratio of a nominator's stake. -pub type PhragmenAssignment = (AccountId, T); - -/// Particular `AccountId` was backed by `ExtendedBalance` of a nominator's stake. -pub type PhragmenStakedAssignment = (AccountId, ExtendedBalance); - /// Final result of the phragmen election. #[derive(RuntimeDebug)] pub struct PhragmenResult { @@ -111,7 +149,142 @@ pub struct PhragmenResult { pub winners: Vec<(AccountId, ExtendedBalance)>, /// Individual assignments. for each tuple, the first elements is a voter and the second /// is the list of candidates that it supports. - pub assignments: Vec<(AccountId, Vec>)> + pub assignments: Vec>, +} + +/// A voter's stake assignment among a set of targets, represented as ratios. +#[derive(RuntimeDebug, Clone, Default)] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] +pub struct Assignment { + /// Voter's identifier + pub who: AccountId, + /// The distribution of the voter's stake. + pub distribution: Vec<(AccountId, T)>, +} + +impl Assignment +where + ExtendedBalance: From<::Inner>, +{ + /// Convert from a ratio assignment into one with absolute values aka. [`StakedAssignment`]. + /// + /// It needs `stake` which is the total budget of the voter. If `fill` is set to true, + /// it _tries_ to ensure that all the potential rounding errors are compensated and the + /// distribution's sum is exactly equal to the total budget, by adding or subtracting the + /// remainder from the last distribution. + /// + /// If an edge ratio is [`Bounded::max_value()`], it is dropped. This edge can never mean + /// anything useful. + pub fn into_staked(self, stake: ExtendedBalance, fill: bool) -> StakedAssignment + where + T: sp_std::ops::Mul, + { + let mut sum: ExtendedBalance = Bounded::min_value(); + let mut distribution = self + .distribution + .into_iter() + .filter_map(|(target, p)| { + // if this ratio is zero, then skip it. + if p == Bounded::min_value() { + None + } else { + // NOTE: this mul impl will always round to the nearest number, so we might both + // overflow and underflow. + let distribution_stake = p * stake; + // defensive only. We assume that balance cannot exceed extended balance. + sum = sum.saturating_add(distribution_stake); + Some((target, distribution_stake)) + } + }) + .collect::>(); + + if fill { + // NOTE: we can do this better. + // https://revs.runtime-revolution.com/getting-100-with-rounded-percentages-273ffa70252b + if let Some(leftover) = stake.checked_sub(sum) { + if let Some(last) = distribution.last_mut() { + last.1 = last.1.saturating_add(leftover); + } + } else if let Some(excess) = sum.checked_sub(stake) { + if let Some(last) = distribution.last_mut() { + last.1 = last.1.saturating_sub(excess); + } + } + } + + StakedAssignment { + who: self.who, + distribution, + } + } +} + +/// A voter's stake assignment among a set of targets, represented as absolute values in the scale +/// of [`ExtendedBalance`]. +#[derive(RuntimeDebug, Clone, Default)] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] +pub struct StakedAssignment { + /// Voter's identifier + pub who: AccountId, + /// The distribution of the voter's stake. + pub distribution: Vec<(AccountId, ExtendedBalance)>, +} + +impl StakedAssignment { + /// Converts self into the normal [`Assignment`] type. + /// + /// If `fill` is set to true, it _tries_ to ensure that all the potential rounding errors are + /// compensated and the distribution's sum is exactly equal to 100%, by adding or subtracting + /// the remainder from the last distribution. + /// + /// NOTE: it is quite critical that this attempt always works. The data type returned here will + /// potentially get used to create a compact type; a compact type requires sum of ratios to be + /// less than 100% upon un-compacting. + /// + /// If an edge stake is so small that it cannot be represented in `T`, it is ignored. This edge + /// can never be re-created and does not mean anything useful anymore. + pub fn into_assignment(self, fill: bool) -> Assignment + where + ExtendedBalance: From<::Inner>, + { + let accuracy: u128 = T::ACCURACY.saturated_into(); + let mut sum: u128 = Zero::zero(); + let stake = self.distribution.iter().map(|x| x.1).sum(); + let mut distribution = self + .distribution + .into_iter() + .filter_map(|(target, w)| { + let per_thing = T::from_rational_approximation(w, stake); + if per_thing == Bounded::min_value() { + None + } else { + sum += per_thing.clone().deconstruct().saturated_into(); + Some((target, per_thing)) + } + }) + .collect::>(); + + if fill { + if let Some(leftover) = accuracy.checked_sub(sum) { + if let Some(last) = distribution.last_mut() { + last.1 = last.1.saturating_add( + T::from_parts(leftover.saturated_into()) + ); + } + } else if let Some(excess) = sum.checked_sub(accuracy) { + if let Some(last) = distribution.last_mut() { + last.1 = last.1.saturating_sub( + T::from_parts(excess.saturated_into()) + ); + } + } + } + + Assignment { + who: self.who, + distribution, + } + } } /// A structure to demonstrate the phragmen result from the perspective of the candidate, i.e. how @@ -122,12 +295,12 @@ pub struct PhragmenResult { /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet /// they do not necessarily have to be the same. #[derive(Default, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, Eq, PartialEq))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Eq, PartialEq))] pub struct Support { /// Total support. pub total: ExtendedBalance, /// Support from voters. - pub voters: Vec>, + pub voters: Vec<(AccountId, ExtendedBalance)>, } /// A linkage from a candidate and its [`Support`]. @@ -164,7 +337,7 @@ pub fn elect( // return structures let mut elected_candidates: Vec<(AccountId, ExtendedBalance)>; - let mut assigned: Vec<(AccountId, Vec>)>; + let mut assigned: Vec>; // used to cache and access candidates index. let mut c_idx_cache = BTreeMap::::new(); @@ -271,7 +444,10 @@ pub fn elect( // update backing stake of candidates and voters for n in &mut voters { - let mut assignment = (n.who.clone(), vec![]); + let mut assignment = Assignment { + who: n.who.clone(), + ..Default::default() + }; for e in &mut n.edges { if elected_candidates.iter().position(|(ref c, _)| *c == e.who).is_some() { let per_bill_parts: R::Inner = @@ -298,44 +474,45 @@ pub fn elect( // R::Inner. .unwrap_or(Bounded::max_value()) } else { - // defensive only. Both edge and nominator loads are built from + // defensive only. Both edge and voter loads are built from // scores, hence MUST have the same denominator. Zero::zero() } } }; let per_thing = R::from_parts(per_bill_parts); - assignment.1.push((e.who.clone(), per_thing)); + assignment.distribution.push((e.who.clone(), per_thing)); } } - if assignment.1.len() > 0 { - // To ensure an assertion indicating: no stake from the nominator going to waste, + let len = assignment.distribution.len(); + if len > 0 { + // To ensure an assertion indicating: no stake from the voter going to waste, // we add a minimal post-processing to equally assign all of the leftover stake ratios. - let vote_count: R::Inner = assignment.1.len().saturated_into(); - let len = assignment.1.len(); - let mut sum: R::Inner = Zero::zero(); - assignment.1.iter().for_each(|a| sum = sum.saturating_add(a.1.deconstruct())); + let vote_count: R::Inner = len.saturated_into(); let accuracy = R::ACCURACY; + let mut sum: R::Inner = Zero::zero(); + assignment.distribution.iter().for_each(|a| sum = sum.saturating_add(a.1.deconstruct())); + let diff = accuracy.saturating_sub(sum); let diff_per_vote = (diff / vote_count).min(accuracy); if !diff_per_vote.is_zero() { for i in 0..len { - let current_ratio = assignment.1[i % len].1; + let current_ratio = assignment.distribution[i % len].1; let next_ratio = current_ratio .saturating_add(R::from_parts(diff_per_vote)); - assignment.1[i % len].1 = next_ratio; + assignment.distribution[i % len].1 = next_ratio; } } - // `remainder` is set to be less than maximum votes of a nominator (currently 16). + // `remainder` is set to be less than maximum votes of a voter (currently 16). // safe to cast it to usize. let remainder = diff - diff_per_vote * vote_count; for i in 0..remainder.saturated_into::() { - let current_ratio = assignment.1[i % len].1; + let current_ratio = assignment.distribution[i % len].1; let next_ratio = current_ratio.saturating_add(R::from_parts(1u8.into())); - assignment.1[i % len].1 = next_ratio; + assignment.distribution[i % len].1 = next_ratio; } assigned.push(assignment); } @@ -347,39 +524,109 @@ pub fn elect( }) } -/// Build the support map from the given phragmen result. -pub fn build_support_map( - elected_stashes: &Vec, - assignments: &Vec<(AccountId, Vec>)>, - stake_of: FS, -) -> SupportMap where +/// Build the support map from the given phragmen result. It maps a flat structure like +/// +/// ```nocompile +/// assignments: vec![ +/// voter1, vec![(candidate1, w11), (candidate2, w12)], +/// voter2, vec![(candidate1, w21), (candidate2, w22)] +/// ] +/// ``` +/// +/// into a mapping of candidates and their respective support: +/// +/// ```nocompile +/// SupportMap { +/// candidate1: Support { +/// own:0, +/// total: w11 + w21, +/// others: vec![(candidate1, w11), (candidate2, w21)] +/// }, +/// candidate2: Support { +/// own:0, +/// total: w12 + w22, +/// others: vec![(candidate1, w12), (candidate2, w22)] +/// }, +/// } +/// ``` +/// +/// The second returned flag indicates the number of edges who didn't corresponded to an actual +/// winner from the given winner set. A value in this place larger than 0 indicates a potentially +/// faulty assignment. +/// +/// `O(E)` where `E` is the total number of edges. +pub fn build_support_map( + winners: &[AccountId], + assignments: &[StakedAssignment], +) -> (SupportMap, u32) where AccountId: Default + Ord + Member, - Balance: Default + Copy + AtLeast32Bit, - C: Convert + Convert, - for<'r> FS: Fn(&'r AccountId) -> Balance, - R: PerThing + sp_std::ops::Mul, { - let to_votes = |b: Balance| >::convert(b) as ExtendedBalance; + let mut errors = 0; // Initialize the support of each candidate. let mut supports = >::new(); - elected_stashes + winners .iter() .for_each(|e| { supports.insert(e.clone(), Default::default()); }); // build support struct. - for (n, assignment) in assignments.iter() { - for (c, per_thing) in assignment.iter() { - let nominator_stake = to_votes(stake_of(n)); - // AUDIT: it is crucially important for the `Mul` implementation of all - // per-things to be sound. - let other_stake = *per_thing * nominator_stake; + for StakedAssignment { who, distribution } in assignments.iter() { + for (c, weight_extended) in distribution.iter() { if let Some(support) = supports.get_mut(c) { - support.voters.push((n.clone(), other_stake)); - support.total = support.total.saturating_add(other_stake); + support.total = support.total.saturating_add(*weight_extended); + support.voters.push((who.clone(), *weight_extended)); + } else { + errors = errors.saturating_add(1); } } } - supports + (supports, errors) +} + +/// Evaluate a phragmen result, given the support map. The returned tuple contains: +/// +/// - Minimum support. This value must be **maximized**. +/// - Sum of all supports. This value must be **maximized**. +/// - Sum of all supports squared. This value must be **minimized**. +/// +/// `O(E)` where `E` is the total number of edges. +pub fn evaluate_support( + support: &SupportMap, +) -> PhragmenScore { + let mut min_support = ExtendedBalance::max_value(); + let mut sum: ExtendedBalance = Zero::zero(); + // NOTE: this will probably saturate but using big num makes it even slower. We'll have to see. + // This must run on chain.. + let mut sum_squared: ExtendedBalance = Zero::zero(); + for (_, support) in support.iter() { + sum += support.total; + let squared = support.total.saturating_mul(support.total); + sum_squared = sum_squared.saturating_add(squared); + if support.total < min_support { + min_support = support.total; + } + } + [min_support, sum, sum_squared] +} + +/// Compares two sets of phragmen scores based on desirability and returns true if `that` is +/// better than `this`. +/// +/// Evaluation is done in a lexicographic manner. +/// +/// Note that the third component should be minimized. +pub fn is_score_better(this: PhragmenScore, that: PhragmenScore) -> bool { + match that + .iter() + .enumerate() + .map(|(i, e)| e.cmp(&this[i])) + .collect::>() + .as_slice() + { + [Ordering::Greater, _, _] => true, + [Ordering::Equal, Ordering::Greater, _] => true, + [Ordering::Equal, Ordering::Equal, Ordering::Less] => true, + _ => false, + } } /// Performs equalize post-processing to the output of the election algorithm. This happens in @@ -388,13 +635,13 @@ pub fn build_support_map( /// /// No value is returned from the function and the `supports` parameter is updated. /// -/// * `assignments`: exactly the same is the output of phragmen. -/// * `supports`: mutable reference to s `SupportMap`. This parameter is updated. -/// * `tolerance`: maximum difference that can occur before an early quite happens. -/// * `iterations`: maximum number of iterations that will be processed. -/// * `stake_of`: something that can return the stake stake of a particular candidate or voter. +/// - `assignments`: exactly the same is the output of phragmen. +/// - `supports`: mutable reference to s `SupportMap`. This parameter is updated. +/// - `tolerance`: maximum difference that can occur before an early quite happens. +/// - `iterations`: maximum number of iterations that will be processed. +/// - `stake_of`: something that can return the stake stake of a particular candidate or voter. pub fn equalize( - mut assignments: Vec<(AccountId, Vec>)>, + mut assignments: Vec>, supports: &mut SupportMap, tolerance: ExtendedBalance, iterations: usize, @@ -408,13 +655,13 @@ pub fn equalize( for _i in 0..iterations { let mut max_diff = 0; - for (voter, assignment) in assignments.iter_mut() { - let voter_budget = stake_of(&voter); + for StakedAssignment { who, distribution } in assignments.iter_mut() { + let voter_budget = stake_of(&who); let diff = do_equalize::<_, _, C>( - voter, + who, voter_budget, - assignment, + distribution, supports, tolerance, ); @@ -432,7 +679,7 @@ pub fn equalize( fn do_equalize( voter: &AccountId, budget_balance: Balance, - elected_edges: &mut Vec>, + elected_edges: &mut Vec<(AccountId, ExtendedBalance)>, support_map: &mut SupportMap, tolerance: ExtendedBalance ) -> ExtendedBalance where diff --git a/primitives/phragmen/src/mock.rs b/primitives/phragmen/src/mock.rs index 66ef64a6c23365de2badbc031dcf2a13357f4576..31ce3d38c3500ef930d3676a141beabb8694b275 100644 --- a/primitives/phragmen/src/mock.rs +++ b/primitives/phragmen/src/mock.rs @@ -18,10 +18,10 @@ #![cfg(test)] -use crate::{elect, PhragmenResult, PhragmenAssignment}; +use crate::{elect, PhragmenResult, Assignment}; use sp_runtime::{ - assert_eq_error_rate, Perbill, PerThing, - traits::{Convert, Member, SaturatedConversion} + assert_eq_error_rate, PerThing, + traits::{Convert, Member, SaturatedConversion, Zero, One} }; use sp_std::collections::btree_map::BTreeMap; @@ -320,22 +320,23 @@ pub(crate) fn create_stake_of(stakes: &[(AccountId, Balance)]) } -pub fn check_assignments(assignments: Vec<(AccountId, Vec>)>) { - for (_, a) in assignments { - let sum: u32 = a.iter().map(|(_, p)| p.deconstruct()).sum(); - assert_eq_error_rate!(sum, Perbill::ACCURACY, 5); +pub fn check_assignments_sum(assignments: Vec>) { + for Assignment { distribution, .. } in assignments { + let mut sum: u128 = Zero::zero(); + distribution.iter().for_each(|(_, p)| sum += p.deconstruct().saturated_into()); + assert_eq_error_rate!(sum, T::ACCURACY.saturated_into(), 1); } } -pub(crate) fn run_and_compare( +pub(crate) fn run_and_compare( candidates: Vec, voters: Vec<(AccountId, Vec)>, - stake_of: Box Balance>, + stake_of: &Box Balance>, to_elect: usize, min_to_elect: usize, ) { // run fixed point code. - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( to_elect, min_to_elect, candidates.clone(), @@ -353,14 +354,14 @@ pub(crate) fn run_and_compare( assert_eq!(winners, truth_value.winners); - for (nominator, assigned) in assignments.clone() { - if let Some(float_assignments) = truth_value.assignments.iter().find(|x| x.0 == nominator) { - for (candidate, per_thingy) in assigned { + for Assignment { who, distribution } in assignments.clone() { + if let Some(float_assignments) = truth_value.assignments.iter().find(|x| x.0 == who) { + for (candidate, per_thingy) in distribution { if let Some(float_assignment) = float_assignments.1.iter().find(|x| x.0 == candidate ) { assert_eq_error_rate!( - Perbill::from_fraction(float_assignment.1).deconstruct(), + Output::from_fraction(float_assignment.1).deconstruct(), per_thingy.deconstruct(), - 1, + Output::Inner::one(), ); } else { panic!("candidate mismatch. This should never happen.") @@ -371,7 +372,7 @@ pub(crate) fn run_and_compare( } } - check_assignments(assignments); + check_assignments_sum(assignments); } pub(crate) fn build_support_map_float( diff --git a/primitives/phragmen/src/node.rs b/primitives/phragmen/src/node.rs new file mode 100644 index 0000000000000000000000000000000000000000..92ef325a348724762f3c92f29849963c1cfe4edb --- /dev/null +++ b/primitives/phragmen/src/node.rs @@ -0,0 +1,287 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! (very) Basic implementation of a graph node used in the reduce algorithm. + +use sp_runtime::RuntimeDebug; +use sp_std::{cell::RefCell, fmt, prelude::*, rc::Rc}; + +/// The role that a node can accept. +#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, RuntimeDebug)] +pub(crate) enum NodeRole { + /// A voter. This is synonym to a nominator in a staking context. + Voter, + /// A target. This is synonym to a candidate/validator in a staking context. + Target, +} + +pub(crate) type RefCellOf = Rc>; +pub(crate) type NodeRef = RefCellOf>; + +/// Identifier of a node. This is particularly handy to have a proper `PartialEq` implementation. +/// Otherwise, self votes wouldn't have been indistinguishable. +#[derive(PartialOrd, Ord, Clone, PartialEq, Eq)] +pub(crate) struct NodeId { + /// An account-like identifier representing the node. + pub who: A, + /// The role of the node. + pub role: NodeRole, +} + +impl NodeId { + /// Create a new [`NodeId`]. + pub fn from(who: A, role: NodeRole) -> Self { + Self { who, role } + } +} + +#[cfg(feature = "std")] +impl sp_std::fmt::Debug for NodeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!( + f, + "Node({:?}, {:?})", + self.who, + if self.role == NodeRole::Voter { + "V" + } else { + "T" + } + ) + } +} + +/// A one-way graph note. This can only store a pointer to its parent. +#[derive(Clone)] +pub(crate) struct Node { + /// The identifier of the note. + pub(crate) id: NodeId, + /// The parent pointer. + pub(crate) parent: Option>, +} + +impl PartialEq for Node { + fn eq(&self, other: &Node) -> bool { + self.id == other.id + } +} + +impl Eq for Node {} + +#[cfg(feature = "std")] +impl fmt::Debug for Node { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "({:?} --> {:?})", + self.id, + self.parent.as_ref().map(|p| p.borrow().id.clone()) + ) + } +} + +impl Node { + /// Create a new [`Node`] + pub fn new(id: NodeId) -> Node { + Self { id, parent: None } + } + + /// Returns true if `other` is the parent of `who`. + pub fn is_parent_of(who: &NodeRef, other: &NodeRef) -> bool { + if who.borrow().parent.is_none() { + return false; + } + who.borrow().parent.as_ref() == Some(other) + } + + /// Removes the parent of `who`. + pub fn remove_parent(who: &NodeRef) { + who.borrow_mut().parent = None; + } + + /// Sets `who`'s parent to be `parent`. + pub fn set_parent_of(who: &NodeRef, parent: &NodeRef) { + who.borrow_mut().parent = Some(parent.clone()); + } + + /// Finds the root of `start`. It return a tuple of `(root, root_vec)` where `root_vec` is the + /// vector of Nodes leading to the root. Hence the first element is the start itself and the + /// last one is the root. As convenient, the root itself is also returned as the first element + /// of the tuple. + /// + /// This function detects cycles and breaks as soon a duplicate node is visited, returning the + /// cycle up to but not including the duplicate node. + /// + /// If you are certain that no cycles exist, you can use [`root_unchecked`]. + pub fn root(start: &NodeRef) -> (NodeRef, Vec>) { + let mut parent_path: Vec> = Vec::new(); + let mut visited: Vec> = Vec::new(); + + parent_path.push(start.clone()); + visited.push(start.clone()); + let mut current = start.clone(); + + while let Some(ref next_parent) = current.clone().borrow().parent { + if visited.contains(next_parent) { + break; + } + parent_path.push(next_parent.clone()); + current = next_parent.clone(); + visited.push(current.clone()); + } + + (current, parent_path) + } + + /// Consumes self and wraps it in a `Rc>`. This type can be used as the pointer type + /// to a parent node. + pub fn into_ref(self) -> NodeRef { + Rc::from(RefCell::from(self)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn id(i: u32) -> NodeId { + NodeId::from(i, NodeRole::Target) + } + + #[test] + fn basic_create_works() { + let node = Node::new(id(10)); + assert_eq!( + node, + Node { + id: NodeId { + who: 10, + role: NodeRole::Target + }, + parent: None + } + ); + } + + #[test] + fn set_parent_works() { + let a = Node::new(id(10)).into_ref(); + let b = Node::new(id(20)).into_ref(); + + assert_eq!(a.borrow().parent, None); + Node::set_parent_of(&a, &b); + assert_eq!(*a.borrow().parent.as_ref().unwrap(), b); + } + + #[test] + fn get_root_singular() { + let a = Node::new(id(1)).into_ref(); + assert_eq!(Node::root(&a), (a.clone(), vec![a.clone()])); + } + + #[test] + fn get_root_works() { + // D <-- A <-- B <-- C + // \ + // <-- E + let a = Node::new(id(1)).into_ref(); + let b = Node::new(id(2)).into_ref(); + let c = Node::new(id(3)).into_ref(); + let d = Node::new(id(4)).into_ref(); + let e = Node::new(id(5)).into_ref(); + let f = Node::new(id(6)).into_ref(); + + Node::set_parent_of(&c, &b); + Node::set_parent_of(&b, &a); + Node::set_parent_of(&e, &a); + Node::set_parent_of(&a, &d); + + assert_eq!( + Node::root(&e), + (d.clone(), vec![e.clone(), a.clone(), d.clone()]), + ); + + assert_eq!(Node::root(&a), (d.clone(), vec![a.clone(), d.clone()]),); + + assert_eq!( + Node::root(&c), + (d.clone(), vec![c.clone(), b.clone(), a.clone(), d.clone()]), + ); + + // D A <-- B <-- C + // F <-- / \ + // <-- E + Node::set_parent_of(&a, &f); + + assert_eq!(Node::root(&a), (f.clone(), vec![a.clone(), f.clone()]),); + + assert_eq!( + Node::root(&c), + (f.clone(), vec![c.clone(), b.clone(), a.clone(), f.clone()]), + ); + } + + #[test] + fn get_root_on_cycle() { + // A ---> B + // | | + // <---- C + let a = Node::new(id(1)).into_ref(); + let b = Node::new(id(2)).into_ref(); + let c = Node::new(id(3)).into_ref(); + + Node::set_parent_of(&a, &b); + Node::set_parent_of(&b, &c); + Node::set_parent_of(&c, &a); + + let (root, path) = Node::root(&a); + assert_eq!(root, c); + assert_eq!(path.clone(), vec![a.clone(), b.clone(), c.clone()]); + } + + #[test] + fn get_root_on_cycle_2() { + // A ---> B + // | | | + // - C + let a = Node::new(id(1)).into_ref(); + let b = Node::new(id(2)).into_ref(); + let c = Node::new(id(3)).into_ref(); + + Node::set_parent_of(&a, &b); + Node::set_parent_of(&b, &c); + Node::set_parent_of(&c, &b); + + let (root, path) = Node::root(&a); + assert_eq!(root, c); + assert_eq!(path.clone(), vec![a.clone(), b.clone(), c.clone()]); + } + + #[test] + fn node_cmp_stack_overflows_on_non_unique_elements() { + // To make sure we don't stack overflow on duplicate who. This needs manual impl of + // PartialEq. + let a = Node::new(id(1)).into_ref(); + let b = Node::new(id(2)).into_ref(); + let c = Node::new(id(3)).into_ref(); + + Node::set_parent_of(&a, &b); + Node::set_parent_of(&b, &c); + Node::set_parent_of(&c, &a); + + Node::root(&a); + } +} diff --git a/primitives/phragmen/src/reduce.rs b/primitives/phragmen/src/reduce.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f6f6c3b991504f1ccd5f9092acbf80870e9d0a6 --- /dev/null +++ b/primitives/phragmen/src/reduce.rs @@ -0,0 +1,1076 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Rust implementation of the Phragmén reduce algorithm. This can be used by any off chain +//! application to reduce cycles from the edge assignment, which will result in smaller size. +//! +//! ### Notions +//! - `m`: size of the committee to elect. +//! - `k`: maximum allowed votes (16 as of this writing). +//! - `nv ∈ E` means that nominator `n ∈ N` supports the election of candidate `v ∈ V`. +//! - A valid solution consists of a tuple `(S, W)` , where `S ⊆ V` is a committee of m validators, +//! and `W : E → R ≥ 0` is an edge weight vector which describes how the budget of each nominator +//! n is fractionally assigned to n 's elected neighbors. +//! - `E_w := { e ∈ E : w_e > 0 }`. +//! +//! ### Algorithm overview +//! +//! > We consider the input edge weight vector `w` as a directed flow over `E_w` , where the flow in +//! > each edge is directed from the nominator to the validator. We build `w′` from `w` by removing +//! > **circulations** to cancel out the flow over as many edges as possible, while preserving flow +//! > demands over all vertices and without reverting the flow direction over any edge. As long as +//! > there is a cycle, we can remove an additional circulation and eliminate at least one new edge +//! > from `E_w′` . This shows that the algorithm always progresses and will eventually finish with +//! > an acyclic edge support. We keep a data structure that represents a forest of rooted trees, +//! > which is initialized as a collection of singletons – one per vertex – and to which edges in +//! > `E_w` are added one by one, causing the trees to merge. Whenever a new edge creates a cycle, +//! > we detect it and destroy it by removing a circulation. We also run a pre-computation which is +//! > designed to detect and remove cycles of length four exclusively. This pre-computation is +//! > optional, and if we skip it then the running time becomes `O (|E_w| ⋅ m), so the +//! > pre-computation makes sense only if `m >> k` and `|E_w| >> m^2`. +//! +//! ### Resources: +//! +//! 1. https://hackmd.io/JOn9x98iS0e0DPWQ87zGWg?view + +use crate::node::{Node, NodeId, NodeRef, NodeRole}; +use crate::{ExtendedBalance, IdentifierT, StakedAssignment}; +use sp_runtime::traits::{Bounded, Zero}; +use sp_std::{ + collections::btree_map::{BTreeMap, Entry::*}, + prelude::*, +}; + +/// Map type used for reduce_4. Can be easily swapped with HashMap. +type Map = BTreeMap<(A, A), A>; + +/// Returns all combinations of size two in the collection `input` with no repetition. +fn combinations_2(input: &[T]) -> Vec<(T, T)> { + let n = input.len(); + if n < 2 { + return Default::default(); + } + + let mut comb = Vec::with_capacity(n * (n - 1) / 2); + for i in 0..n { + for j in i + 1..n { + comb.push((input[i].clone(), input[j].clone())) + } + } + comb +} + +/// Returns the count of trailing common elements in two slices. +pub(crate) fn trailing_common(t1: &[T], t2: &[T]) -> usize { + t1.iter().rev().zip(t2.iter().rev()).take_while(|e| e.0 == e.1).count() +} + +/// Merges two parent roots as described by the reduce algorithm. +fn merge(voter_root_path: Vec>, target_root_path: Vec>) { + let (shorter_path, longer_path) = if voter_root_path.len() <= target_root_path.len() { + (voter_root_path, target_root_path) + } else { + (target_root_path, voter_root_path) + }; + + // iterate from last to beginning, skipping the first one. This asserts that + // indexing is always correct. + shorter_path + .iter() + .zip(shorter_path.iter().skip(1)) + .for_each(|(voter, next)| Node::set_parent_of(&next, &voter)); + Node::set_parent_of(&shorter_path[0], &longer_path[0]); +} + +/// Reduce only redundant edges with cycle length of 4. +/// +/// Returns the number of edges removed. +/// +/// It is strictly assumed that the `who` attribute of all provided assignments are unique. The +/// result will most likely be corrupt otherwise. +/// +/// O(|E_w| ⋅ k). +fn reduce_4(assignments: &mut Vec>) -> u32 { + let mut combination_map: Map = Map::new(); + let mut num_changed: u32 = Zero::zero(); + + // we have to use the old fashioned loops here with manual indexing. Borrowing assignments will + // not work since then there is NO way to mutate it inside. + for assignment_index in 0..assignments.len() { + let who = assignments[assignment_index].who.clone(); + + // all combinations for this particular voter + let distribution_ids = &assignments[assignment_index] + .distribution + .iter() + .map(|(t, _p)| t.clone()) + .collect::>(); + let candidate_combinations = combinations_2(distribution_ids); + + for (v1, v2) in candidate_combinations { + match combination_map.entry((v1.clone(), v2.clone())) { + Vacant(entry) => { + entry.insert(who.clone()); + } + Occupied(mut entry) => { + let other_who = entry.get_mut(); + + // double check if who is still voting for this pair. If not, it means that this + // pair is no longer valid and must have been removed in previous rounds. The + // reason for this is subtle; candidate_combinations is created once while the + // inner loop might remove some edges. Note that if count() > 2, the we have + // duplicates. + if assignments[assignment_index] + .distribution + .iter() + .filter(|(t, _)| *t == v1 || *t == v2) + .count() != 2 + { + continue; + } + + // check if other_who voted for the same pair v1, v2. + let maybe_other_assignments = assignments.iter().find(|a| a.who == *other_who); + if maybe_other_assignments.is_none() { + continue; + } + let other_assignment = + maybe_other_assignments.expect("value is checked to be 'Some'"); + + // Collect potential cycle votes + let mut other_cycle_votes = other_assignment + .distribution + .iter() + .filter_map(|(t, w)| { + if *t == v1 || *t == v2 { + Some((t.clone(), *w)) + } else { + None + } + }) + .collect::>(); + + let other_votes_count = other_cycle_votes.len(); + + // If the length is more than 2, then we have identified duplicates. For now, we + // just skip. Later on we can early exit and stop processing this data since it + // is corrupt anyhow. + debug_assert!(other_votes_count <= 2); + + if other_votes_count < 2 { + // This is not a cycle. Replace and continue. + *other_who = who.clone(); + continue; + } else if other_votes_count == 2 { + // This is a cycle. + let mut who_cycle_votes: Vec<(A, ExtendedBalance)> = Vec::with_capacity(2); + assignments[assignment_index] + .distribution + .iter() + .for_each(|(t, w)| { + if *t == v1 || *t == v2 { + who_cycle_votes.push((t.clone(), *w)); + } + }); + + if who_cycle_votes.len() != 2 { + continue; + } + + // Align the targets similarly. This helps with the circulation below. + if other_cycle_votes[0].0 != who_cycle_votes[0].0 { + other_cycle_votes.swap(0, 1); + } + + // Find min + let mut min_value: ExtendedBalance = Bounded::max_value(); + let mut min_index: usize = 0; + let cycle = who_cycle_votes + .iter() + .chain(other_cycle_votes.iter()) + .enumerate() + .map(|(index, (t, w))| { + if *w <= min_value { + min_value = *w; + min_index = index; + } + (t.clone(), *w) + }) + .collect::>(); + + // min was in the first part of the chained iters + let mut increase_indices: Vec = Vec::new(); + let mut decrease_indices: Vec = Vec::new(); + decrease_indices.push(min_index); + if min_index < 2 { + // min_index == 0 => sibling_index <- 1 + // min_index == 1 => sibling_index <- 0 + let sibling_index = 1 - min_index; + increase_indices.push(sibling_index); + // valid because the two chained sections of `cycle` are aligned; + // index [0, 2] are both voting for v1 or both v2. Same goes for [1, 3]. + decrease_indices.push(sibling_index + 2); + increase_indices.push(min_index + 2); + } else { + // min_index == 2 => sibling_index <- 3 + // min_index == 3 => sibling_index <- 2 + let sibling_index = 3 - min_index % 2; + increase_indices.push(sibling_index); + // valid because the two chained sections of `cycle` are aligned; + // index [0, 2] are both voting for v1 or both v2. Same goes for [1, 3]. + decrease_indices.push(sibling_index - 2); + increase_indices.push(min_index - 2); + } + + // apply changes + let mut remove_indices: Vec = Vec::with_capacity(1); + increase_indices.into_iter().for_each(|i| { + let voter = if i < 2 { + who.clone() + } else { + other_who.clone() + }; + // Note: so this is pretty ambiguous. We should only look for one + // assignment that meets this criteria and if we find multiple then that + // is a corrupt input. Same goes for the next block. + assignments + .iter_mut() + .filter(|a| a.who == voter) + .for_each(|ass| { + ass.distribution + .iter_mut() + .position(|(t, _)| *t == cycle[i].0) + .map(|idx| { + let next_value = + ass.distribution[idx].1.saturating_add(min_value); + ass.distribution[idx].1 = next_value; + }); + }); + }); + decrease_indices.into_iter().for_each(|i| { + let voter = if i < 2 { + who.clone() + } else { + other_who.clone() + }; + assignments + .iter_mut() + .filter(|a| a.who == voter) + .for_each(|ass| { + ass.distribution + .iter_mut() + .position(|(t, _)| *t == cycle[i].0) + .map(|idx| { + let next_value = + ass.distribution[idx].1.saturating_sub(min_value); + if next_value.is_zero() { + ass.distribution.remove(idx); + remove_indices.push(i); + num_changed += 1; + } else { + ass.distribution[idx].1 = next_value; + } + }); + }); + }); + + // remove either one of them. + let who_removed = remove_indices.iter().find(|i| **i < 2usize).is_some(); + let other_removed = + remove_indices.into_iter().find(|i| *i >= 2usize).is_some(); + + match (who_removed, other_removed) { + (false, true) => { + *other_who = who.clone(); + } + (true, false) => { + // nothing, other_who can stay there. + } + (true, true) => { + // remove and don't replace + entry.remove(); + } + (false, false) => { + // Neither of the edges was removed? impossible. + panic!("Duplicate voter (or other corrupt input)."); + } + } + } + } + } + } + } + + num_changed +} + +/// Reduce redundant edges from the edge weight graph, with all possible length. +/// +/// To get the best performance, this should be called after `reduce_4()`. +/// +/// Returns the number of edges removed. +/// +/// It is strictly assumed that the `who` attribute of all provided assignments are unique. The +/// result will most likely be corrupt otherwise. +/// +/// O(|Ew| ⋅ m) +fn reduce_all(assignments: &mut Vec>) -> u32 { + let mut num_changed: u32 = Zero::zero(); + let mut tree: BTreeMap, NodeRef> = BTreeMap::new(); + + // NOTE: This code can heavily use an index cache. Looking up a pair of (voter, target) in the + // assignments happens numerous times and and we can save time. For now it is written as such + // because abstracting some of this code into a function/closure is super hard due to borrow + // checks (and most likely needs unsafe code at the end). For now I will keep it as it and + // refactor later. + + // a flat iterator of (voter, target) over all pairs of votes. Similar to reduce_4, we loop + // without borrowing. + for assignment_index in 0..assignments.len() { + let voter = assignments[assignment_index].who.clone(); + + let mut dist_index = 0; + loop { + // A distribution could have been removed. We don't know for sure. Hence, we check. + let maybe_dist = assignments[assignment_index].distribution.get(dist_index); + if maybe_dist.is_none() { + // The rest of this loop is moot. + break; + } + let (target, _) = maybe_dist.expect("Value checked to be some").clone(); + + // store if they existed already. + let voter_id = NodeId::from(voter.clone(), NodeRole::Voter); + let target_id = NodeId::from(target.clone(), NodeRole::Target); + let voter_exists = tree.contains_key(&voter_id); + let target_exists = tree.contains_key(&target_id); + + // create both. + let voter_node = tree + .entry(voter_id.clone()) + .or_insert(Node::new(voter_id).into_ref()) + .clone(); + let target_node = tree + .entry(target_id.clone()) + .or_insert(Node::new(target_id).into_ref()) + .clone(); + + // If one exists but the other one doesn't, or if both does not, then set the existing + // one as the parent of the non-existing one and move on. Else, continue with the rest + // of the code. + match (voter_exists, target_exists) { + (false, false) => { + Node::set_parent_of(&target_node, &voter_node); + dist_index += 1; + continue; + } + (false, true) => { + Node::set_parent_of(&voter_node, &target_node); + dist_index += 1; + continue; + } + (true, false) => { + Node::set_parent_of(&target_node, &voter_node); + dist_index += 1; + continue; + } + (true, true) => { /* don't continue and execute the rest */ } + }; + + let (voter_root, voter_root_path) = Node::root(&voter_node); + let (target_root, target_root_path) = Node::root(&target_node); + + if voter_root != target_root { + // swap + merge(voter_root_path, target_root_path); + dist_index += 1; + } else { + // find common and cycle. + let common_count = trailing_common(&voter_root_path, &target_root_path); + + // because roots are the same. + #[cfg(feature = "std")] + debug_assert_eq!( + target_root_path.last().unwrap(), + voter_root_path.last().unwrap() + ); + debug_assert!(common_count > 0); + + // cycle part of each path will be `path[path.len() - common_count - 1 : 0]` + // NOTE: the order of chaining is important! it is always build from [target, ..., + // voter] + let cycle = target_root_path + .iter() + .take(target_root_path.len() - common_count + 1) + .cloned() + .chain( + voter_root_path + .iter() + .take(voter_root_path.len() - common_count) + .rev() + .cloned(), + ) + .collect::>>(); + + // a cycle's length shall always be multiple of two. + #[cfg(feature = "std")] + debug_assert_eq!(cycle.len() % 2, 0); + + // find minimum of cycle. + let mut min_value: ExtendedBalance = Bounded::max_value(); + // The voter and the target pair that create the min edge. + let mut min_target: A = Default::default(); + let mut min_voter: A = Default::default(); + // The index of the min in opaque cycle list. + let mut min_index = 0usize; + // 1 -> next // 0 -> prev + let mut min_direction = 0u32; + // helpers + let next_index = |i| { + if i < (cycle.len() - 1) { + i + 1 + } else { + 0 + } + }; + let prev_index = |i| { + if i > 0 { + i - 1 + } else { + cycle.len() - 1 + } + }; + for i in 0..cycle.len() { + if cycle[i].borrow().id.role == NodeRole::Voter { + // NOTE: sadly way too many clones since I don't want to make A: Copy + let current = cycle[i].borrow().id.who.clone(); + let next = cycle[next_index(i)].borrow().id.who.clone(); + let prev = cycle[prev_index(i)].borrow().id.who.clone(); + assignments.iter().find(|a| a.who == current).map(|ass| { + ass.distribution.iter().find(|d| d.0 == next).map(|(_, w)| { + if *w < min_value { + min_value = *w; + min_target = next.clone(); + min_voter = current.clone(); + min_index = i; + min_direction = 1; + } + }) + }); + assignments.iter().find(|a| a.who == current).map(|ass| { + ass.distribution.iter().find(|d| d.0 == prev).map(|(_, w)| { + if *w < min_value { + min_value = *w; + min_target = prev.clone(); + min_voter = current.clone(); + min_index = i; + min_direction = 0; + } + }) + }); + } + } + + // if the min edge is in the voter's sub-chain. + // [target, ..., X, Y, ... voter] + let target_chunk = target_root_path.len() - common_count; + let min_chain_in_voter = (min_index + min_direction as usize) > target_chunk; + + // walk over the cycle and update the weights + let mut should_inc_counter = true; + let start_operation_add = ((min_index % 2) + min_direction as usize) % 2 == 1; + let mut additional_removed = Vec::new(); + for i in 0..cycle.len() { + let current = cycle[i].borrow(); + if current.id.role == NodeRole::Voter { + let prev = cycle[prev_index(i)].borrow(); + assignments + .iter_mut() + .enumerate() + .filter(|(_, a)| a.who == current.id.who) + .for_each(|(target_ass_index, ass)| { + ass.distribution + .iter_mut() + .position(|(t, _)| *t == prev.id.who) + .map(|idx| { + let next_value = if i % 2 == 0 { + if start_operation_add { + ass.distribution[idx].1.saturating_add(min_value) + } else { + ass.distribution[idx].1.saturating_sub(min_value) + } + } else { + if start_operation_add { + ass.distribution[idx].1.saturating_sub(min_value) + } else { + ass.distribution[idx].1.saturating_add(min_value) + } + }; + + if next_value.is_zero() { + // if the removed edge is from the current assignment, dis_index + // should NOT be increased. + if target_ass_index == assignment_index { + should_inc_counter = false + } + ass.distribution.remove(idx); + num_changed += 1; + // only add if this is not the min itself. + if !(i == min_index && min_direction == 0) { + additional_removed.push(( + cycle[i].clone(), + cycle[prev_index(i)].clone(), + )); + } + } else { + ass.distribution[idx].1 = next_value; + } + }); + }); + + let next = cycle[next_index(i)].borrow(); + assignments + .iter_mut() + .enumerate() + .filter(|(_, a)| a.who == current.id.who) + .for_each(|(target_ass_index, ass)| { + ass.distribution + .iter_mut() + .position(|(t, _)| *t == next.id.who) + .map(|idx| { + let next_value = if i % 2 == 0 { + if start_operation_add { + ass.distribution[idx].1.saturating_sub(min_value) + } else { + ass.distribution[idx].1.saturating_add(min_value) + } + } else { + if start_operation_add { + ass.distribution[idx].1.saturating_add(min_value) + } else { + ass.distribution[idx].1.saturating_sub(min_value) + } + }; + + if next_value.is_zero() { + // if the removed edge is from the current assignment, dis_index + // should NOT be increased. + if target_ass_index == assignment_index { + should_inc_counter = false + } + ass.distribution.remove(idx); + num_changed += 1; + if !(i == min_index && min_direction == 1) { + additional_removed.push(( + cycle[i].clone(), + cycle[next_index(i)].clone(), + )); + } + } else { + ass.distribution[idx].1 = next_value; + } + }); + }); + } + } + + // don't do anything if the edge removed itself. This is always the first and last + // element + let should_reorg = !(min_index == (cycle.len() - 1) && min_direction == 1); + + // re-org. + if should_reorg { + let min_edge = vec![min_voter, min_target]; + if min_chain_in_voter { + // NOTE: safe; voter_root_path is always bigger than 1 element. + for i in 0..voter_root_path.len() - 1 { + let current = voter_root_path[i].clone().borrow().id.who.clone(); + let next = voter_root_path[i + 1].clone().borrow().id.who.clone(); + if min_edge.contains(¤t) && min_edge.contains(&next) { + break; + } + Node::set_parent_of(&voter_root_path[i + 1], &voter_root_path[i]); + } + Node::set_parent_of(&voter_node, &target_node); + } else { + // NOTE: safe; target_root_path is always bigger than 1 element. + for i in 0..target_root_path.len() - 1 { + let current = target_root_path[i].clone().borrow().id.who.clone(); + let next = target_root_path[i + 1].clone().borrow().id.who.clone(); + if min_edge.contains(¤t) && min_edge.contains(&next) { + break; + } + Node::set_parent_of(&target_root_path[i + 1], &target_root_path[i]); + } + Node::set_parent_of(&target_node, &voter_node); + } + } + + // remove every other node which has collapsed to zero + for (r1, r2) in additional_removed { + if Node::is_parent_of(&r1, &r2) { + Node::remove_parent(&r1); + } else if Node::is_parent_of(&r2, &r1) { + Node::remove_parent(&r2); + } + } + + // increment the counter if needed. + if should_inc_counter { + dist_index += 1; + } + } + } + } + + num_changed +} + +/// Reduce the given [`PhragmenResult`]. This removes redundant edges from without changing the +/// overall backing of any of the elected candidates. +/// +/// Returns the number of edges removed. +/// +/// IMPORTANT: It is strictly assumed that the `who` attribute of all provided assignments are +/// unique. The result will most likely be corrupt otherwise. Furthermore, if the _distribution +/// vector_ contains duplicate ids, only the first instance is ever updates. +/// +/// O(min{ |Ew| ⋅ k + m3 , |Ew| ⋅ m }) +pub fn reduce(assignments: &mut Vec>) -> u32 where { + let mut num_changed = reduce_4(assignments); + num_changed += reduce_all(assignments); + num_changed +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merging_works() { + // D <-- A <-- B <-- C + // + // F <-- E + let d = Node::new(NodeId::from(1, NodeRole::Target)).into_ref(); + let a = Node::new(NodeId::from(2, NodeRole::Target)).into_ref(); + let b = Node::new(NodeId::from(3, NodeRole::Target)).into_ref(); + let c = Node::new(NodeId::from(4, NodeRole::Target)).into_ref(); + let e = Node::new(NodeId::from(5, NodeRole::Target)).into_ref(); + let f = Node::new(NodeId::from(6, NodeRole::Target)).into_ref(); + + Node::set_parent_of(&c, &b); + Node::set_parent_of(&b, &a); + Node::set_parent_of(&a, &d); + Node::set_parent_of(&e, &f); + + let path1 = vec![c.clone(), b.clone(), a.clone(), d.clone()]; + let path2 = vec![e.clone(), f.clone()]; + + merge(path1, path2); + // D <-- A <-- B <-- C + // | + // F --> E --> --> + assert_eq!(e.borrow().clone().parent.unwrap().borrow().id.who, 4u32); // c + } + + #[test] + fn merge_with_len_one() { + // D <-- A <-- B <-- C + // + // F <-- E + let d = Node::new(NodeId::from(1, NodeRole::Target)).into_ref(); + let a = Node::new(NodeId::from(2, NodeRole::Target)).into_ref(); + let b = Node::new(NodeId::from(3, NodeRole::Target)).into_ref(); + let c = Node::new(NodeId::from(4, NodeRole::Target)).into_ref(); + let f = Node::new(NodeId::from(6, NodeRole::Target)).into_ref(); + + Node::set_parent_of(&c, &b); + Node::set_parent_of(&b, &a); + Node::set_parent_of(&a, &d); + + let path1 = vec![c.clone(), b.clone(), a.clone(), d.clone()]; + let path2 = vec![f.clone()]; + + merge(path1, path2); + // D <-- A <-- B <-- C + // | + // F --> --> + assert_eq!(f.borrow().clone().parent.unwrap().borrow().id.who, 4u32); // c + } + + #[test] + fn basic_reduce_4_cycle_works() { + use super::*; + + let assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 25), (20, 75)], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 50), (20, 50)], + }, + ]; + + let mut new_assignments = assignments.clone(); + let num_reduced = reduce_4(&mut new_assignments); + + assert_eq!(num_reduced, 1); + assert_eq!( + new_assignments, + vec![ + StakedAssignment { + who: 1, + distribution: vec![(20, 100),], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 75), (20, 25),], + }, + ], + ); + } + + #[test] + fn basic_reduce_all_cycles_works() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10)], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5)], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 15), (40, 15)], + }, + StakedAssignment { + who: 4, + distribution: vec![(20, 10), (30, 10), (40, 20)], + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 20), (30, 10), (40, 20)], + }, + ]; + + assert_eq!(3, reduce_all(&mut assignments)); + + assert_eq!( + assignments, + vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10),] + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5),], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 30),], + }, + StakedAssignment { + who: 4, + distribution: vec![(40, 40),] + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 15), (30, 20), (40, 15),], + }, + ], + ) + } + + #[test] + fn basic_reduce_works() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10)], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5)], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 15), (40, 15)], + }, + StakedAssignment { + who: 4, + distribution: vec![(20, 10), (30, 10), (40, 20)], + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 20), (30, 10), (40, 20)], + }, + ]; + + assert_eq!(3, reduce(&mut assignments)); + + assert_eq!( + assignments, + vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10),] + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5),], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 30),], + }, + StakedAssignment { + who: 4, + distribution: vec![(40, 40),] + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 15), (30, 20), (40, 15),], + }, + ], + ) + } + + #[test] + fn should_deal_with_self_vote() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10)], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5)], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 15), (40, 15)], + }, + StakedAssignment { + who: 4, + distribution: vec![(20, 10), (30, 10), (40, 20)], + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 20), (30, 10), (40, 20)], + }, + // self vote from 10 and 20 to itself. + StakedAssignment { + who: 10, + distribution: vec![(10, 100)], + }, + StakedAssignment { + who: 20, + distribution: vec![(20, 200)], + }, + ]; + + assert_eq!(3, reduce(&mut assignments)); + + assert_eq!( + assignments, + vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10),] + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 5),], + }, + StakedAssignment { + who: 3, + distribution: vec![(20, 30),], + }, + StakedAssignment { + who: 4, + distribution: vec![(40, 40),] + }, + StakedAssignment { + who: 5, + distribution: vec![(20, 15), (30, 20), (40, 15),], + }, + // should stay untouched. + StakedAssignment { + who: 10, + distribution: vec![(10, 100)] + }, + StakedAssignment { + who: 20, + distribution: vec![(20, 200)] + }, + ], + ) + } + + #[test] + fn reduce_3_common_votes_same_weight() { + let mut assignments = vec![ + StakedAssignment { + who: 4, + distribution: vec![ + ( + 1000000, + 100, + ), + ( + 1000002, + 100, + ), + ( + 1000004, + 100, + ), + ], + }, + StakedAssignment { + who: 5, + distribution: vec![ + ( + 1000000, + 100, + ), + ( + 1000002, + 100, + ), + ( + 1000004, + 100, + ), + ], + }, + ]; + + reduce_4(&mut assignments); + + assert_eq!( + assignments, + vec![ + StakedAssignment { + who: 4, + distribution: vec![(1000000, 200,), (1000004, 100,),], + }, + StakedAssignment { + who: 5, + distribution: vec![(1000002, 200,), (1000004, 100,),], + }, + ], + ) + } + + #[test] + #[should_panic] + fn reduce_panics_on_duplicate_voter() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 10), (20, 10)], + }, + StakedAssignment { + who: 1, + distribution: vec![(10, 15), (20, 5)], + }, + StakedAssignment { + who: 2, + distribution: vec![(10, 15), (20, 15)], + }, + ]; + + reduce(&mut assignments); + } + + #[test] + fn should_deal_with_duplicates_target() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 15), (20, 5)], + }, + StakedAssignment { + who: 2, + distribution: vec![ + (10, 15), + (20, 15), + // duplicate + (10, 1), + // duplicate + (20, 1), + ], + }, + ]; + + reduce(&mut assignments); + + assert_eq!( + assignments, + vec![ + StakedAssignment { + who: 1, + distribution: vec![(10, 20),], + }, + StakedAssignment { + who: 2, + distribution: vec![ + (10, 10), + (20, 20), + // duplicate votes are silently ignored. + (10, 1), + (20, 1), + ], + }, + ], + ) + } + + #[test] + fn bound_should_be_kept() { + let mut assignments = vec![ + StakedAssignment { + who: 1, + distribution: vec![(103, 72), (101, 53), (100, 83), (102, 38)], + }, + StakedAssignment { + who: 2, + distribution: vec![(103, 18), (101, 36), (102, 54), (100, 94)], + }, + StakedAssignment { + who: 3, + distribution: vec![(100, 96), (101, 35), (102, 52), (103, 69)], + }, + StakedAssignment { + who: 4, + distribution: vec![(102, 34), (100, 47), (103, 91), (101, 73)], + }, + ]; + + let winners = vec![103, 101, 100, 102]; + + let n = 4; + let m = winners.len() as u32; + let num_reduced = reduce_all(&mut assignments); + assert!(16 - num_reduced <= n + m); + } +} diff --git a/primitives/phragmen/src/tests.rs b/primitives/phragmen/src/tests.rs index 8bcf007c7b4c7d29b04b0ad115380825989cdc3a..e9861ede723cf9f31baf5f6c5d91b79e361b75d9 100644 --- a/primitives/phragmen/src/tests.rs +++ b/primitives/phragmen/src/tests.rs @@ -19,11 +19,12 @@ #![cfg(test)] use crate::mock::*; -use crate::{elect, PhragmenResult, PhragmenStakedAssignment, build_support_map, Support, equalize}; +use crate::{ + elect, equalize, build_support_map, is_score_better, + Support, StakedAssignment, Assignment, PhragmenResult, ExtendedBalance, +}; use substrate_test_utils::assert_eq_uvec; -use sp_runtime::Perbill; - -type Output = Perbill; +use sp_runtime::{Perbill, Permill, Percent, PerU16, traits::Convert}; #[test] fn float_phragmen_poc_works() { @@ -81,7 +82,7 @@ fn phragmen_poc_works() { ]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates, @@ -92,9 +93,21 @@ fn phragmen_poc_works() { assert_eq_uvec!( assignments, vec![ - (10, vec![(2, Perbill::from_percent(100))]), - (20, vec![(3, Perbill::from_percent(100))]), - (30, vec![(2, Perbill::from_percent(100/2)), (3, Perbill::from_percent(100/2))]), + Assignment { + who: 10u64, + distribution: vec![(2, Perbill::from_percent(100))], + }, + Assignment { + who: 20, + distribution: vec![(3, Perbill::from_percent(100))], + }, + Assignment { + who: 30, + distribution: vec![ + (2, Perbill::from_percent(100/2)), + (3, Perbill::from_percent(100/2)), + ], + }, ] ); } @@ -115,7 +128,10 @@ fn phragmen_poc_2_works() { (4, 500), ]); - run_and_compare(candidates, voters, stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates, voters, &stake_of, 2, 2); } #[test] @@ -133,7 +149,10 @@ fn phragmen_poc_3_works() { (4, 1000), ]); - run_and_compare(candidates, voters, stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); + run_and_compare::(candidates, voters, &stake_of, 2, 2); } #[test] @@ -149,7 +168,7 @@ fn phragmen_accuracy_on_large_scale_only_validators() { (5, (u64::max_value() - 2).into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates.clone(), @@ -158,7 +177,7 @@ fn phragmen_accuracy_on_large_scale_only_validators() { assert_eq_uvec!(winners, vec![(1, 18446744073709551614u128), (5, 18446744073709551613u128)]); assert_eq!(assignments.len(), 2); - check_assignments(assignments); + check_assignments_sum(assignments); } #[test] @@ -179,7 +198,7 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { (14, u64::max_value().into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates, @@ -190,13 +209,25 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { assert_eq!( assignments, vec![ - (13, vec![(1, Perbill::one())]), - (14, vec![(2, Perbill::one())]), - (1, vec![(1, Perbill::one())]), - (2, vec![(2, Perbill::one())]), + Assignment { + who: 13u64, + distribution: vec![(1, Perbill::one())], + }, + Assignment { + who: 14, + distribution: vec![(2, Perbill::one())], + }, + Assignment { + who: 1, + distribution: vec![(1, Perbill::one())], + }, + Assignment { + who: 2, + distribution: vec![(2, Perbill::one())], + }, ] ); - check_assignments(assignments); + check_assignments_sum(assignments); } #[test] @@ -210,7 +241,7 @@ fn phragmen_accuracy_on_small_scale_self_vote() { (30, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( 3, 3, candidates, @@ -240,7 +271,7 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( 3, 3, candidates, @@ -273,7 +304,7 @@ fn phragmen_large_scale_test() { (50, 990000000000000000), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates, @@ -281,7 +312,7 @@ fn phragmen_large_scale_test() { ).unwrap(); assert_eq_uvec!(winners, vec![(24, 1490000000000200000u128), (22, 1490000000000100000u128)]); - check_assignments(assignments); + check_assignments_sum(assignments); } #[test] @@ -299,7 +330,7 @@ fn phragmen_large_scale_test_2() { (50, nom_budget.into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates, @@ -310,12 +341,24 @@ fn phragmen_large_scale_test_2() { assert_eq!( assignments, vec![ - (50, vec![(2, Perbill::from_parts(500000001)), (4, Perbill::from_parts(499999999))]), - (2, vec![(2, Perbill::one())]), - (4, vec![(4, Perbill::one())]), + Assignment { + who: 50u64, + distribution: vec![ + (2, Perbill::from_parts(500000001)), + (4, Perbill::from_parts(499999999)) + ], + }, + Assignment { + who: 2, + distribution: vec![(2, Perbill::one())], + }, + Assignment { + who: 4, + distribution: vec![(4, Perbill::one())], + }, ], ); - check_assignments(assignments); + check_assignments_sum(assignments); } #[test] @@ -348,7 +391,7 @@ fn phragmen_linear_equalize() { (130, 1000), ]); - run_and_compare(candidates, voters, stake_of, 2, 2); + run_and_compare::(candidates, voters, &stake_of, 2, 2); } #[test] @@ -363,7 +406,7 @@ fn elect_has_no_entry_barrier() { (2, 10), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( 3, 3, candidates, @@ -390,7 +433,7 @@ fn minimum_to_elect_is_respected() { (2, 10), ]); - let maybe_result = elect::<_, _, TestCurrencyToVote, Output>( + let maybe_result = elect::<_, _, TestCurrencyToVote, Perbill>( 10, 10, candidates, @@ -416,7 +459,7 @@ fn self_votes_should_be_kept() { (1, 8), ]); - let result = elect::<_, _, TestCurrencyToVote, Output>( + let result = elect::<_, _, TestCurrencyToVote, Perbill>( 2, 2, candidates, @@ -427,27 +470,28 @@ fn self_votes_should_be_kept() { assert_eq!( result.assignments, vec![ - (10, vec![(10, Perbill::from_percent(100))]), - (20, vec![(20, Perbill::from_percent(100))]), - (1, vec![ + Assignment { who: 10, distribution: vec![(10, Perbill::from_percent(100))] }, + Assignment { who: 20, distribution: vec![(20, Perbill::from_percent(100))] }, + Assignment { who: 1, distribution: vec![ (10, Perbill::from_percent(50)), (20, Perbill::from_percent(50)) ] - ) + }, ], ); - let mut supports = build_support_map::< - Balance, - AccountId, - _, - TestCurrencyToVote, - Output, - >( - &result.winners.into_iter().map(|(who, _)| who).collect(), - &result.assignments, - &stake_of - ); + let staked_assignments: Vec> = result.assignments + .into_iter() + .map(|a| { + let stake = >::convert(stake_of(&a.who)) as ExtendedBalance; + a.into_staked(stake, true) + }).collect(); + + let winners = result.winners.into_iter().map(|(who, _)| who).collect::>(); + let (mut supports, _) = build_support_map::( + winners.as_slice(), + &staked_assignments, + ); assert_eq!(supports.get(&5u64), None); assert_eq!( @@ -459,23 +503,13 @@ fn self_votes_should_be_kept() { &Support { total: 24u128, voters: vec![(20u64, 20u128), (1u64, 4u128)] }, ); - let assignments = result.assignments; - let mut staked_assignments - : Vec<(AccountId, Vec>)> - = Vec::with_capacity(assignments.len()); - for (n, assignment) in assignments.iter() { - let mut staked_assignment - : Vec> - = Vec::with_capacity(assignment.len()); - let stake = stake_of(&n); - for (c, per_thing) in assignment.iter() { - let vote_stake = *per_thing * stake; - staked_assignment.push((c.clone(), vote_stake)); - } - staked_assignments.push((n.clone(), staked_assignment)); - } - - equalize::(staked_assignments, &mut supports, 0, 2usize, &stake_of); + equalize::( + staked_assignments, + &mut supports, + 0, + 2usize, + &stake_of, + ); assert_eq!( supports.get(&10u64).unwrap(), @@ -486,3 +520,462 @@ fn self_votes_should_be_kept() { &Support { total: 20u128, voters: vec![(20u64, 20u128)] }, ); } + +#[test] +fn assignment_convert_works() { + let staked = StakedAssignment { + who: 1 as AccountId, + distribution: vec![ + (20, 100 as Balance), + (30, 25), + ], + }; + + let assignment = staked.clone().into_assignment(true); + assert_eq!( + assignment, + Assignment { + who: 1, + distribution: vec![ + (20, Perbill::from_percent(80)), + (30, Perbill::from_percent(20)), + ] + } + ); + + assert_eq!( + assignment.into_staked(125, true), + staked, + ); +} + +#[test] +fn score_comparison_is_lexicographical() { + // only better in the fist parameter, worse in the other two ✅ + assert_eq!( + is_score_better([10, 20, 30], [12, 10, 35]), + true, + ); + + // worse in the first, better in the other two ❌ + assert_eq!( + is_score_better([10, 20, 30], [9, 30, 10]), + false, + ); + + // equal in the first, the second one dictates. + assert_eq!( + is_score_better([10, 20, 30], [10, 25, 40]), + true, + ); + + // equal in the first two, the last one dictates. + assert_eq!( + is_score_better([10, 20, 30], [10, 20, 40]), + false, + ); +} + +mod compact { + use codec::{Decode, Encode}; + use crate::generate_compact_solution_type; + use super::{AccountId, Balance}; + // these need to come from the same dev-dependency `sp-phragmen`, not from the crate. + use sp_phragmen::{Assignment, StakedAssignment, Error as PhragmenError}; + use sp_std::{convert::{TryInto, TryFrom}, fmt::Debug}; + use sp_runtime::Percent; + + type Accuracy = Percent; + + generate_compact_solution_type!(TestCompact, 16); + + #[test] + fn compact_struct_is_codec() { + let compact = TestCompact::<_, _, _> { + votes1: vec![(2u64, 20), (4, 40)], + votes2: vec![ + (1, (10, Accuracy::from_percent(80)), 11), + (5, (50, Accuracy::from_percent(85)), 51), + ], + ..Default::default() + }; + + let encoded = compact.encode(); + + assert_eq!( + compact, + Decode::decode(&mut &encoded[..]).unwrap(), + ); + } + + fn basic_ratio_test_with() where + V: codec::Codec + Copy + Default + PartialEq + Eq + TryInto + TryFrom + From + Debug, + T: codec::Codec + Copy + Default + PartialEq + Eq + TryInto + TryFrom + From + Debug, + >::Error: std::fmt::Debug, + >::Error: std::fmt::Debug, + >::Error: std::fmt::Debug, + >::Error: std::fmt::Debug, + { + let voters = vec![ + 2 as AccountId, + 4, + 1, + 5, + 3, + ]; + let targets = vec![ + 10 as AccountId, + 11, + 20, // 2 + 30, + 31, // 4 + 32, + 40, // 6 + 50, + 51, // 8 + ]; + + let assignments = vec![ + Assignment { + who: 2 as AccountId, + distribution: vec![(20u64, Accuracy::from_percent(100))] + }, + Assignment { + who: 4, + distribution: vec![(40, Accuracy::from_percent(100))], + }, + Assignment { + who: 1, + distribution: vec![ + (10, Accuracy::from_percent(80)), + (11, Accuracy::from_percent(20)) + ], + }, + Assignment { + who: 5, + distribution: vec![ + (50, Accuracy::from_percent(85)), + (51, Accuracy::from_percent(15)), + ] + }, + Assignment { + who: 3, + distribution: vec![ + (30, Accuracy::from_percent(50)), + (31, Accuracy::from_percent(25)), + (32, Accuracy::from_percent(25)), + ], + }, + ]; + + let voter_index = |a: &AccountId| -> Option { + voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + let target_index = |a: &AccountId| -> Option { + targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + + let compacted = >::from_assignment( + assignments.clone(), + voter_index, + target_index, + ).unwrap(); + + assert_eq!( + compacted, + TestCompact { + votes1: vec![(V::from(0u8), T::from(2u8)), (V::from(1u8), T::from(6u8))], + votes2: vec![ + (V::from(2u8), (T::from(0u8), Accuracy::from_percent(80)), T::from(1u8)), + (V::from(3u8), (T::from(7u8), Accuracy::from_percent(85)), T::from(8u8)), + ], + votes3: vec![ + ( + V::from(4), + [(T::from(3u8), Accuracy::from_percent(50)), (T::from(4u8), Accuracy::from_percent(25))], + T::from(5u8), + ), + ], + ..Default::default() + } + ); + + let voter_at = |a: V| -> Option { voters.get(>::try_into(a).unwrap()).cloned() }; + let target_at = |a: T| -> Option { targets.get(>::try_into(a).unwrap()).cloned() }; + + assert_eq!( + compacted.into_assignment(voter_at, target_at).unwrap(), + assignments, + ); + } + + #[test] + fn basic_from_and_into_compact_works_assignments() { + basic_ratio_test_with::(); + basic_ratio_test_with::(); + basic_ratio_test_with::(); + } + + #[test] + fn basic_from_and_into_compact_works_staked_assignments() { + let voters = vec![ + 2 as AccountId, + 4, + 1, + 5, + 3, + ]; + let targets = vec![ + 10 as AccountId, 11, + 20, + 30, 31, 32, + 40, + 50, 51, + ]; + + let assignments = vec![ + StakedAssignment { + who: 2 as AccountId, + distribution: vec![(20, 100 as Balance)] + }, + StakedAssignment { + who: 4, + distribution: vec![(40, 100)], + }, + StakedAssignment { + who: 1, + distribution: vec![ + (10, 80), + (11, 20) + ], + }, + StakedAssignment { + who: 5, distribution: + vec![ + (50, 85), + (51, 15), + ] + }, + StakedAssignment { + who: 3, + distribution: vec![ + (30, 50), + (31, 25), + (32, 25), + ], + }, + ]; + + let voter_index = |a: &AccountId| -> Option { + voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + let target_index = |a: &AccountId| -> Option { + targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + + let compacted = >::from_staked( + assignments.clone(), + voter_index, + target_index, + ).unwrap(); + + assert_eq!( + compacted, + TestCompact { + votes1: vec![(0, 2), (1, 6)], + votes2: vec![ + (2, (0, 80), 1), + (3, (7, 85), 8), + ], + votes3: vec![ + (4, [(3, 50), (4, 25)], 5), + ], + ..Default::default() + } + ); + + let max_of_fn = |_: &AccountId| -> Balance { 100u128 }; + let voter_at = |a: u16| -> Option { voters.get(a as usize).cloned() }; + let target_at = |a: u16| -> Option { targets.get(a as usize).cloned() }; + + assert_eq!( + compacted.into_staked( + max_of_fn, + voter_at, + target_at, + ).unwrap(), + assignments, + ); + } + + #[test] + fn compact_into_stake_must_report_overflow() { + // The last edge which is computed from the rest should ALWAYS be positive. + // in votes2 + let compact = TestCompact:: { + votes1: Default::default(), + votes2: vec![(0, (1, 10), 2)], + ..Default::default() + }; + + let entity_at = |a: u16| -> Option { Some(a as AccountId) }; + let max_of = |_: &AccountId| -> Balance { 5 }; + + assert_eq!( + compact.into_staked(&max_of, &entity_at, &entity_at).unwrap_err(), + PhragmenError::CompactStakeOverflow, + ); + + // in votes3 onwards + let compact = TestCompact:: { + votes1: Default::default(), + votes2: Default::default(), + votes3: vec![(0, [(1, 7), (2, 8)], 3)], + ..Default::default() + }; + + assert_eq!( + compact.into_staked(&max_of, &entity_at, &entity_at).unwrap_err(), + PhragmenError::CompactStakeOverflow, + ); + + // Also if equal + let compact = TestCompact:: { + votes1: Default::default(), + votes2: Default::default(), + // 5 is total, we cannot leave none for 30 here. + votes3: vec![(0, [(1, 3), (2, 2)], 3)], + ..Default::default() + }; + + assert_eq!( + compact.into_staked(&max_of, &entity_at, &entity_at).unwrap_err(), + PhragmenError::CompactStakeOverflow, + ); + } + + #[test] + fn compact_into_assignment_must_report_overflow() { + // in votes2 + let compact = TestCompact:: { + votes1: Default::default(), + votes2: vec![(0, (1, Accuracy::from_percent(100)), 2)], + ..Default::default() + }; + + let entity_at = |a: u16| -> Option { Some(a as AccountId) }; + + assert_eq!( + compact.into_assignment(&entity_at, &entity_at).unwrap_err(), + PhragmenError::CompactStakeOverflow, + ); + + // in votes3 onwards + let compact = TestCompact:: { + votes1: Default::default(), + votes2: Default::default(), + votes3: vec![(0, [(1, Accuracy::from_percent(70)), (2, Accuracy::from_percent(80))], 3)], + ..Default::default() + }; + + assert_eq!( + compact.into_assignment(&entity_at, &entity_at).unwrap_err(), + PhragmenError::CompactStakeOverflow, + ); + } + + #[test] + fn target_count_overflow_is_detected() { + let assignments = vec![ + StakedAssignment { + who: 1 as AccountId, + distribution: (10..26).map(|i| (i as AccountId, i as Balance)).collect::>(), + }, + ]; + + let entity_index = |a: &AccountId| -> Option { Some(*a as u16) }; + + let compacted = >::from_staked( + assignments.clone(), + entity_index, + entity_index, + ); + + assert!(compacted.is_ok()); + + let assignments = vec![ + StakedAssignment { + who: 1 as AccountId, + distribution: (10..27).map(|i| (i as AccountId, i as Balance)).collect::>(), + }, + ]; + + let compacted = >::from_staked( + assignments.clone(), + entity_index, + entity_index, + ); + + assert_eq!( + compacted.unwrap_err(), + PhragmenError::CompactTargetOverflow, + ); + + let assignments = vec![ + Assignment { + who: 1 as AccountId, + distribution: (10..27).map(|i| (i as AccountId, Percent::from_parts(i as u8))).collect::>(), + }, + ]; + + let compacted = >::from_assignment( + assignments.clone(), + entity_index, + entity_index, + ); + + assert_eq!( + compacted.unwrap_err(), + PhragmenError::CompactTargetOverflow, + ); + } + + #[test] + fn zero_target_count_is_ignored() { + let voters = vec![1 as AccountId, 2]; + let targets = vec![10 as AccountId, 11]; + + let assignments = vec![ + StakedAssignment { + who: 1 as AccountId, + distribution: vec![(10, 100 as Balance), (11, 100)] + }, + StakedAssignment { + who: 2, + distribution: vec![], + }, + ]; + + let voter_index = |a: &AccountId| -> Option { + voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + let target_index = |a: &AccountId| -> Option { + targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + + let compacted = >::from_staked( + assignments.clone(), + voter_index, + target_index, + ).unwrap(); + + assert_eq!( + compacted, + TestCompact { + votes1: Default::default(), + votes2: vec![(0, (0, 100), 1)], + ..Default::default() + } + ); + } +} diff --git a/primitives/rpc/Cargo.toml b/primitives/rpc/Cargo.toml index 5631f7a891f0d174530caada2c2d99dd447f6cb9..a7975fb4754b9ee049b7f76f45ca0876a4b91006 100644 --- a/primitives/rpc/Cargo.toml +++ b/primitives/rpc/Cargo.toml @@ -14,3 +14,6 @@ sp-core = { version = "2.0.0-alpha.5", path = "../core" } [dev-dependencies] serde_json = "1.0.41" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/Cargo.toml b/primitives/runtime-interface/Cargo.toml index 68da7f1bf8009bf535e876ccf968aa5771218778..01f7f2f19c98169dbebd86e841fd37473ea6d011 100644 --- a/primitives/runtime-interface/Cargo.toml +++ b/primitives/runtime-interface/Cargo.toml @@ -14,7 +14,7 @@ sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../wasm-interface", def sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-runtime-interface-proc-macro = { version = "2.0.0-alpha.5", path = "proc-macro" } sp-externalities = { version = "0.8.0-alpha.5", optional = true, path = "../externalities" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } static_assertions = "1.0.0" primitive-types = { version = "0.7.0", default-features = false } @@ -43,3 +43,6 @@ std = [ # Disables static assertions in `impls.rs` that checks the word size. To prevent any footgun, the # check is changed into a runtime check. disable_target_static_assertions = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/proc-macro/Cargo.toml b/primitives/runtime-interface/proc-macro/Cargo.toml index c8c12b9bdbde21d6a15289beceba6aeabefc4d3a..6d0b7ee5fb7262c4a55922c0b1cef6549424c935 100644 --- a/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/primitives/runtime-interface/proc-macro/Cargo.toml @@ -18,3 +18,6 @@ quote = "1.0.3" proc-macro2 = "1.0.3" Inflector = "0.11.4" proc-macro-crate = "0.1.4" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml index 128f6c9acc4f0d047f73b1eb8b37a80d843487bc..6f2d66bd77c4b2d672c54d3dc6f760830ccc570b 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml +++ b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml @@ -10,10 +10,10 @@ repository = "https://github.com/paritytech/substrate/" publish = false [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.2", default-features = false, path = "../" } -sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../std" } -sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../io" } -sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../" } +sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } +sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../io" } +sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } @@ -21,3 +21,6 @@ wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-run [features] default = [ "std" ] std = [ "sp-runtime-interface/std", "sp-std/std", "sp-core/std", "sp-io/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/test-wasm/Cargo.toml b/primitives/runtime-interface/test-wasm/Cargo.toml index cf72a10749591957db0b4bb99396f102481e3989..4eb4f01c9f0c810ab42f9016e1f7db66147e1f65 100644 --- a/primitives/runtime-interface/test-wasm/Cargo.toml +++ b/primitives/runtime-interface/test-wasm/Cargo.toml @@ -21,3 +21,6 @@ wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-run [features] default = [ "std" ] std = [ "sp-runtime-interface/std", "sp-std/std", "sp-core/std", "sp-io/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml index 18cff357d8aa1f582ceaa95184b6fcc480ab917d..f3bee038c8204c612444c8c3d6e34a4e7f3e2126 100644 --- a/primitives/runtime-interface/test/Cargo.toml +++ b/primitives/runtime-interface/test/Cargo.toml @@ -16,3 +16,6 @@ sp-runtime-interface-test-wasm-deprecated = { version = "2.0.0-dev", path = "../ sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } sp-io = { version = "2.0.0-alpha.5", path = "../../io" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index 743748fc108156a4526d3bbf2c00591dd3fc16cf..0e71e8becd61e4ea18a77e9ae4b5afcc031d731d 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/sp-runtime" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } sp-arithmetic = { version = "2.0.0-alpha.5", default-features = false, path = "../arithmetic" } @@ -32,6 +32,7 @@ rand = "0.7.2" [features] bench = [] +runtime-benchmarks = [] default = ["std"] std = [ "sp-application-crypto/std", @@ -47,3 +48,6 @@ std = [ "parity-util-mem/std", "hash256-std-hasher/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime/src/curve.rs b/primitives/runtime/src/curve.rs index e04ce77fb2cbb7dc1d5ed7d028e621799138153c..b00cbed6525a16cf6f1db35eed8f5b6e225437f3 100644 --- a/primitives/runtime/src/curve.rs +++ b/primitives/runtime/src/curve.rs @@ -16,7 +16,7 @@ //! Provides some utilities to define a piecewise linear function. -use crate::{Perbill, PerThing, traits::{AtLeast32Bit, SaturatedConversion}}; +use crate::{Perbill, traits::{AtLeast32Bit, SaturatedConversion}}; use core::ops::Sub; /// Piecewise Linear function in [0, 1] -> [0, 1]. diff --git a/primitives/runtime/src/generic/checked_extrinsic.rs b/primitives/runtime/src/generic/checked_extrinsic.rs index 25a8274354ae2aa60be8a97ff368aec2782abb2b..a329f334c0d7707edc8a4e07ada8eac443accce2 100644 --- a/primitives/runtime/src/generic/checked_extrinsic.rs +++ b/primitives/runtime/src/generic/checked_extrinsic.rs @@ -18,10 +18,9 @@ //! stage. use crate::traits::{ - self, Member, MaybeDisplay, SignedExtension, Dispatchable, + self, Member, MaybeDisplay, SignedExtension, Dispatchable, DispatchInfoOf, ValidateUnsigned, }; -use crate::traits::ValidateUnsigned; -use crate::transaction_validity::TransactionValidity; +use crate::transaction_validity::{TransactionValidity, TransactionSource}; /// Definition of something that the external world might want to say; its /// existence implies that it has been checked and is good, particularly with @@ -36,47 +35,53 @@ pub struct CheckedExtrinsic { pub function: Call, } -impl traits::Applyable for +impl traits::Applyable for CheckedExtrinsic where AccountId: Member + MaybeDisplay, Call: Member + Dispatchable, - Extra: SignedExtension, + Extra: SignedExtension, Origin: From>, - Info: Clone, { type Call = Call; - type DispatchInfo = Info; fn validate>( &self, - info: Self::DispatchInfo, + // TODO [#5006;ToDr] should source be passed to `SignedExtension`s? + // Perhaps a change for 2.0 to avoid breaking too much APIs? + source: TransactionSource, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { if let Some((ref id, ref extra)) = self.signed { - Extra::validate(extra, id, &self.function, info.clone(), len) + Extra::validate(extra, id, &self.function, info, len) } else { let valid = Extra::validate_unsigned(&self.function, info, len)?; - let unsigned_validation = U::validate_unsigned(&self.function)?; + let unsigned_validation = U::validate_unsigned(source, &self.function)?; Ok(valid.combine_with(unsigned_validation)) } } fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> crate::ApplyExtrinsicResult { let (maybe_who, pre) = if let Some((id, extra)) = self.signed { - let pre = Extra::pre_dispatch(extra, &id, &self.function, info.clone(), len)?; + let pre = Extra::pre_dispatch(extra, &id, &self.function, info, len)?; (Some(id), pre) } else { - let pre = Extra::pre_dispatch_unsigned(&self.function, info.clone(), len)?; + let pre = Extra::pre_dispatch_unsigned(&self.function, info, len)?; U::pre_dispatch(&self.function)?; (None, pre) }; let res = self.function.dispatch(Origin::from(maybe_who)); - Extra::post_dispatch(pre, info.clone(), len); - Ok(res.map_err(Into::into)) + let post_info = match res { + Ok(info) => info, + Err(err) => err.post_info, + }; + let res = res.map(|_| ()).map_err(|e| e.error); + Extra::post_dispatch(pre, info, &post_info, len, &res)?; + Ok(res) } } diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index a516bc1f7fa999f1087718f93f072ff8eff45099..3e9e52ba8bedac6417bcf5f9a49b79a172923229 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -357,7 +357,6 @@ mod tests { type AccountId = u64; type Call = (); type AdditionalSigned = (); - type DispatchInfo = (); type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 632deb13cbb45af6bd2271c2a5073b6ab05f0619..c80971b576df2d9c6312bb3df0966ebc942cc680 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -68,7 +68,10 @@ pub use sp_application_crypto::{RuntimeAppPublic, BoundToRuntimeAppPublic}; pub use sp_core::RuntimeDebug; /// Re-export top-level arithmetic stuff. -pub use sp_arithmetic::{Perquintill, Perbill, Permill, Percent, Rational128, Fixed64, PerThing}; +pub use sp_arithmetic::{ + Perquintill, Perbill, Permill, Percent, PerU16, Rational128, Fixed64, PerThing, + traits::SaturatedConversion, +}; /// Re-export 128 bit helpers. pub use sp_arithmetic::helpers_128bit; /// Re-export big_uint stuff. @@ -350,10 +353,15 @@ impl From for DispatchOutcome { } } -/// Result of a module function call; either nothing (functions are only called for "side effects") -/// or an error message. +/// This is the legacy return type of `Dispatchable`. It is still exposed for compatibilty +/// reasons. The new return type is `DispatchResultWithInfo`. +/// FRAME runtimes should use frame_support::dispatch::DispatchResult pub type DispatchResult = sp_std::result::Result<(), DispatchError>; +/// Return type of a `Dispatchable` which contains the `DispatchResult` and additional information +/// about the `Dispatchable` that is only known post dispatch. +pub type DispatchResultWithInfo = sp_std::result::Result>; + /// Reason why a dispatch call failed #[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize))] @@ -376,6 +384,18 @@ pub enum DispatchError { }, } +/// Result of a `Dispatchable` which contains the `DispatchResult` and additional information +/// about the `Dispatchable` that is only known post dispatch. +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, RuntimeDebug)] +pub struct DispatchErrorWithPostInfo where + Info: Eq + PartialEq + Clone + Copy + Encode + Decode + traits::Printable +{ + /// Addditional information about the `Dispatchable` which is only known post dispatch. + pub post_info: Info, + /// The actual `DispatchResult` indicating whether the dispatch was succesfull. + pub error: DispatchError, +} + impl DispatchError { /// Return the same error but without the attached message. pub fn stripped(self) -> Self { @@ -387,6 +407,18 @@ impl DispatchError { } } +impl From for DispatchErrorWithPostInfo where + T: Eq + PartialEq + Clone + Copy + Encode + Decode + traits::Printable + Default, + E: Into +{ + fn from(error: E) -> Self { + Self { + post_info: Default::default(), + error: error.into(), + } + } +} + impl From for DispatchError { fn from(_: crate::traits::LookupError) -> Self { Self::CannotLookup @@ -416,6 +448,14 @@ impl From for &'static str { } } +impl From> for &'static str where + T: Eq + PartialEq + Clone + Copy + Encode + Decode + traits::Printable +{ + fn from(err: DispatchErrorWithPostInfo) -> &'static str { + err.error.into() + } +} + impl traits::Printable for DispatchError { fn print(&self) { "DispatchError".print(); @@ -434,6 +474,16 @@ impl traits::Printable for DispatchError { } } +impl traits::Printable for DispatchErrorWithPostInfo where + T: Eq + PartialEq + Clone + Copy + Encode + Decode + traits::Printable +{ + fn print(&self) { + self.error.print(); + "PostInfo: ".print(); + self.post_info.print(); + } +} + /// This type specifies the outcome of dispatching a call to a module. /// /// In case of failure an error specific to the module is returned. diff --git a/primitives/runtime/src/testing.rs b/primitives/runtime/src/testing.rs index 24390700542112b7083b17025e01ec167e43a995..1414a5f4f0a753d02abc6ed2c25c574bfbe531f7 100644 --- a/primitives/runtime/src/testing.rs +++ b/primitives/runtime/src/testing.rs @@ -21,13 +21,13 @@ use std::{fmt::Debug, ops::Deref, fmt, cell::RefCell}; use crate::codec::{Codec, Encode, Decode}; use crate::traits::{ self, Checkable, Applyable, BlakeTwo256, OpaqueKeys, - SignedExtension, Dispatchable, + SignedExtension, Dispatchable, DispatchInfoOf, }; use crate::traits::ValidateUnsigned; use crate::{generic, KeyTypeId, ApplyExtrinsicResult}; pub use sp_core::{H256, sr25519}; use sp_core::{crypto::{CryptoType, Dummy, key_types, Public}, U256}; -use crate::transaction_validity::{TransactionValidity, TransactionValidityError}; +use crate::transaction_validity::{TransactionValidity, TransactionValidityError, TransactionSource}; /// Authority Id #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, Debug, Hash, Serialize, Deserialize, PartialOrd, Ord)] @@ -345,19 +345,18 @@ impl traits::Extrinsic for TestXt } } -impl Applyable for TestXt where +impl Applyable for TestXt where Call: 'static + Sized + Send + Sync + Clone + Eq + Codec + Debug + Dispatchable, - Extra: SignedExtension, + Extra: SignedExtension, Origin: From>, - Info: Clone, { type Call = Call; - type DispatchInfo = Info; /// Checks to see if this is a valid *transaction*. It returns information on it if so. fn validate>( &self, - _info: Self::DispatchInfo, + _source: TransactionSource, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(Default::default()) @@ -367,7 +366,7 @@ impl Applyable for TestXt where /// index and sender. fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> ApplyExtrinsicResult { let maybe_who = if let Some((who, extra)) = self.signature { @@ -378,6 +377,6 @@ impl Applyable for TestXt where None }; - Ok(self.call.dispatch(maybe_who.into()).map_err(Into::into)) + Ok(self.call.dispatch(maybe_who.into()).map(|_| ()).map_err(|e| e.error)) } } diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index 81b7733319bf84e9e5b7af9c45cda2d1ddcb3a27..fdf1d6396d26f84f54d891240fff310c966aa1c9 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -17,7 +17,7 @@ //! Primitives for the runtime modules. use sp_std::prelude::*; -use sp_std::{self, result, marker::PhantomData, convert::{TryFrom, TryInto}, fmt::Debug}; +use sp_std::{self, marker::PhantomData, convert::{TryFrom, TryInto}, fmt::Debug}; use sp_io; #[cfg(feature = "std")] use std::fmt::Display; @@ -28,7 +28,8 @@ use serde::{Serialize, Deserialize, de::DeserializeOwned}; use sp_core::{self, Hasher, TypeId, RuntimeDebug}; use crate::codec::{Codec, Encode, Decode}; use crate::transaction_validity::{ - ValidTransaction, TransactionValidity, TransactionValidityError, UnknownTransaction, + ValidTransaction, TransactionSource, TransactionValidity, TransactionValidityError, + UnknownTransaction, }; use crate::generic::{Digest, DigestItem}; pub use sp_arithmetic::traits::{ @@ -38,6 +39,7 @@ pub use sp_arithmetic::traits::{ }; use sp_application_crypto::AppKey; use impl_trait_for_tuples::impl_for_tuples; +use crate::DispatchResult; /// A lazy value. pub trait Lazy { @@ -146,18 +148,6 @@ impl From for &'static str { } } -/// Some sort of check on the origin is performed by this object. -pub trait EnsureOrigin { - /// A return type. - type Success; - /// Perform the origin check. - fn ensure_origin(o: OuterOrigin) -> result::Result { - Self::try_origin(o).map_err(|_| BadOrigin) - } - /// Perform the origin check. - fn try_origin(o: OuterOrigin) -> result::Result; -} - /// An error that indicates that a lookup failed. #[derive(Encode, Decode, RuntimeDebug)] pub struct LookupError; @@ -329,51 +319,6 @@ impl > SimpleBitOps for T {} -/// The block finalization trait. Implementing this lets you express what should happen -/// for your module when the block is ending. -#[impl_for_tuples(30)] -pub trait OnFinalize { - /// The block is being finalized. Implement to have something happen. - fn on_finalize(_n: BlockNumber) {} -} - -/// The block initialization trait. Implementing this lets you express what should happen -/// for your module when the block is beginning (right before the first extrinsic is executed). -#[impl_for_tuples(30)] -pub trait OnInitialize { - /// The block is being initialized. Implement to have something happen. - fn on_initialize(_n: BlockNumber) {} -} - -/// The runtime upgrade trait. Implementing this lets you express what should happen -/// when the runtime upgrades, and changes may need to occur to your module. -#[impl_for_tuples(30)] -pub trait OnRuntimeUpgrade { - /// Perform a module upgrade. - fn on_runtime_upgrade() {} -} - -/// Off-chain computation trait. -/// -/// Implementing this trait on a module allows you to perform long-running tasks -/// that make (by default) validators generate transactions that feed results -/// of those long-running computations back on chain. -/// -/// NOTE: This function runs off-chain, so it can access the block state, -/// but cannot preform any alterations. More specifically alterations are -/// not forbidden, but they are not persisted in any way after the worker -/// has finished. -#[impl_for_tuples(30)] -pub trait OffchainWorker { - /// This function is being called after every block import (when fully synced). - /// - /// Implement this and use any of the `Offchain` `sp_io` set of APIs - /// to perform off-chain computations, calls and submit transactions - /// with results to trigger any on-chain changes. - /// Any state alterations are lost and are not persisted. - fn offchain_worker(_n: BlockNumber) {} -} - /// Abstraction around hashing // Stupid bug in the Rust compiler believes derived // traits must be fulfilled by all type parameters. @@ -680,8 +625,30 @@ pub trait Dispatchable { type Origin; /// ... type Trait; - /// Actually dispatch this call and result the result of it. - fn dispatch(self, origin: Self::Origin) -> crate::DispatchResult; + /// An opaque set of information attached to the transaction. This could be constructed anywhere + /// down the line in a runtime. The current Substrate runtime uses a struct with the same name + /// to represent the dispatch class and weight. + type Info; + /// Additional information that is returned by `dispatch`. Can be used to supply the caller + /// with information about a `Dispatchable` that is ownly known post dispatch. + type PostInfo: Eq + PartialEq + Clone + Copy + Encode + Decode + Printable; + /// Actually dispatch this call and return the result of it. + fn dispatch(self, origin: Self::Origin) -> crate::DispatchResultWithInfo; +} + +/// Shortcut to reference the `Info` type of a `Dispatchable`. +pub type DispatchInfoOf = ::Info; +/// Shortcut to reference the `PostInfo` type of a `Dispatchable`. +pub type PostDispatchInfoOf = ::PostInfo; + +impl Dispatchable for () { + type Origin = (); + type Trait = (); + type Info = (); + type PostInfo = (); + fn dispatch(self, _origin: Self::Origin) -> crate::DispatchResultWithInfo { + panic!("This implemention should not be used for actual dispatch."); + } } /// Means by which a transaction may be extended. This type embodies both the data and the logic @@ -697,7 +664,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq type AccountId; /// The type which encodes the call to be dispatched. - type Call; + type Call: Dispatchable; /// Any additional data that will go into the signed payload. This may be created dynamically /// from the transaction using the `additional_signed` function. @@ -706,11 +673,6 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// The type that encodes information that can be passed from pre_dispatch to post-dispatch. type Pre: Default; - /// An opaque set of information attached to the transaction. This could be constructed anywhere - /// down the line in a runtime. The current Substrate runtime uses a struct with the same name - /// to represent the dispatch class and weight. - type DispatchInfo: Clone; - /// Construct any additional data that should be in the signed payload of the transaction. Can /// also perform any pre-signature-verification checks and return an error if needed. fn additional_signed(&self) -> Result; @@ -728,7 +690,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq &self, _who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(ValidTransaction::default()) @@ -746,7 +708,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq self, who: &Self::AccountId, call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { self.validate(who, call, info.clone(), len) @@ -764,7 +726,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// Make sure to perform the same checks in `pre_dispatch_unsigned` function. fn validate_unsigned( _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(ValidTransaction::default()) @@ -780,7 +742,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// perform the same validation as in `validate_unsigned`. fn pre_dispatch_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { Self::validate_unsigned(call, info.clone(), len) @@ -788,8 +750,28 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq .map_err(Into::into) } - /// Do any post-flight stuff for a transaction. - fn post_dispatch(_pre: Self::Pre, _info: Self::DispatchInfo, _len: usize) { } + /// Do any post-flight stuff for an extrinsic. + /// + /// This gets given the `DispatchResult` `_result` from the extrinsic and can, if desired, + /// introduce a `TransactionValidityError`, causing the block to become invalid for including + /// it. + /// + /// WARNING: It is dangerous to return an error here. To do so will fundamentally invalidate the + /// transaction and any block that it is included in, causing the block author to not be + /// compensated for their work in validating the transaction or producing the block so far. + /// + /// It can only be used safely when you *know* that the extrinsic is one that can only be + /// introduced by the current block author; generally this implies that it is an inherent and + /// will come from either an offchain-worker or via `InherentData`. + fn post_dispatch( + _pre: Self::Pre, + _info: &DispatchInfoOf, + _post_info: &PostDispatchInfoOf, + _len: usize, + _result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + Ok(()) + } /// Returns the list of unique identifier for this signed extension. /// @@ -804,11 +786,10 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq } #[impl_for_tuples(1, 12)] -impl SignedExtension for Tuple { - for_tuples!( where #( Tuple: SignedExtension )* ); +impl SignedExtension for Tuple { + for_tuples!( where #( Tuple: SignedExtension )* ); type AccountId = AccountId; type Call = Call; - type DispatchInfo = Info; const IDENTIFIER: &'static str = "You should call `identifier()`!"; for_tuples!( type AdditionalSigned = ( #( Tuple::AdditionalSigned ),* ); ); for_tuples!( type Pre = ( #( Tuple::Pre ),* ); ); @@ -821,44 +802,47 @@ impl SignedExtension for Tuple { &self, who: &Self::AccountId, call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { let valid = ValidTransaction::default(); - for_tuples!( #( let valid = valid.combine_with(Tuple.validate(who, call, info.clone(), len)?); )* ); + for_tuples!( #( let valid = valid.combine_with(Tuple.validate(who, call, info, len)?); )* ); Ok(valid) } - fn pre_dispatch(self, who: &Self::AccountId, call: &Self::Call, info: Self::DispatchInfo, len: usize) + fn pre_dispatch(self, who: &Self::AccountId, call: &Self::Call, info: &DispatchInfoOf, len: usize) -> Result { - Ok(for_tuples!( ( #( Tuple.pre_dispatch(who, call, info.clone(), len)? ),* ) )) + Ok(for_tuples!( ( #( Tuple.pre_dispatch(who, call, info, len)? ),* ) )) } fn validate_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { let valid = ValidTransaction::default(); - for_tuples!( #( let valid = valid.combine_with(Tuple::validate_unsigned(call, info.clone(), len)?); )* ); + for_tuples!( #( let valid = valid.combine_with(Tuple::validate_unsigned(call, info, len)?); )* ); Ok(valid) } fn pre_dispatch_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { - Ok(for_tuples!( ( #( Tuple::pre_dispatch_unsigned(call, info.clone(), len)? ),* ) )) + Ok(for_tuples!( ( #( Tuple::pre_dispatch_unsigned(call, info, len)? ),* ) )) } fn post_dispatch( pre: Self::Pre, - info: Self::DispatchInfo, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, len: usize, - ) { - for_tuples!( #( Tuple::post_dispatch(pre.Tuple, info.clone(), len); )* ) + result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + for_tuples!( #( Tuple::post_dispatch(pre.Tuple, info, post_info, len, result)?; )* ); + Ok(()) } fn identifier() -> Vec<&'static str> { @@ -875,7 +859,6 @@ impl SignedExtension for () { type AdditionalSigned = (); type Call = (); type Pre = (); - type DispatchInfo = (); const IDENTIFIER: &'static str = "UnitSignedExtension"; fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } } @@ -888,15 +871,13 @@ impl SignedExtension for () { /// each piece of attributable information to be disambiguated. pub trait Applyable: Sized + Send + Sync { /// Type by which we can dispatch. Restricts the `UnsignedValidator` type. - type Call; - - /// An opaque set of information attached to the transaction. - type DispatchInfo: Clone; + type Call: Dispatchable; /// Checks to see if this is a valid *transaction*. It returns information on it if so. fn validate>( &self, - info: Self::DispatchInfo, + source: TransactionSource, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity; @@ -904,7 +885,7 @@ pub trait Applyable: Sized + Send + Sync { /// index and sender. fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> crate::ApplyExtrinsicResult; } @@ -942,7 +923,7 @@ pub trait ValidateUnsigned { /// /// Changes made to storage WILL be persisted if the call returns `Ok`. fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { - Self::validate_unsigned(call) + Self::validate_unsigned(TransactionSource::InBlock, call) .map(|_| ()) .map_err(Into::into) } @@ -953,7 +934,7 @@ pub trait ValidateUnsigned { /// whether the transaction would panic if it were included or not. /// /// Changes made to storage should be discarded by caller. - fn validate_unsigned(call: &Self::Call) -> TransactionValidity; + fn validate_unsigned(source: TransactionSource, call: &Self::Call) -> TransactionValidity; } /// Opaque data type that may be destructured into a series of raw byte slices (which represent @@ -966,7 +947,7 @@ pub trait OpaqueKeys: Clone { fn key_ids() -> &'static [crate::KeyTypeId]; /// Get the raw bytes of key with key-type ID `i`. fn get_raw(&self, i: super::KeyTypeId) -> &[u8]; - /// Get the decoded key with index `i`. + /// Get the decoded key with key-type ID `i`. fn get(&self, i: super::KeyTypeId) -> Option { T::decode(&mut self.get_raw(i)).ok() } @@ -1303,6 +1284,12 @@ impl Printable for bool { } } +impl Printable for () { + fn print(&self) { + "()".print() + } +} + #[impl_for_tuples(1, 12)] impl Printable for Tuple { fn print(&self) { diff --git a/primitives/runtime/src/transaction_validity.rs b/primitives/runtime/src/transaction_validity.rs index 6dfb80b6d7e4568b0bb0edabe51592e0506261c9..95903b4876262d29b97d7362782eed07290aa4cf 100644 --- a/primitives/runtime/src/transaction_validity.rs +++ b/primitives/runtime/src/transaction_validity.rs @@ -52,6 +52,13 @@ pub enum InvalidTransaction { ExhaustsResources, /// Any other custom invalid validity that is not covered by this enum. Custom(u8), + /// An extrinsic with a Mandatory dispatch resulted in Error. This is indicative of either a + /// malicious validator or a buggy `provide_inherent`. In any case, it can result in dangerously + /// overweight blocks and therefore if found, invalidates the block. + BadMandatory, + /// A transaction with a mandatory dispatch. This is invalid; only inherent extrinsics are + /// allowed to have mandatory dispatches. + MandatoryDispatch, } impl InvalidTransaction { @@ -62,6 +69,14 @@ impl InvalidTransaction { _ => false, } } + + /// Returns if the reason for the invalidity was a mandatory call failing. + pub fn was_mandatory(&self) -> bool { + match self { + Self::BadMandatory => true, + _ => false, + } + } } impl From for &'static str { @@ -76,6 +91,10 @@ impl From for &'static str { "Transaction would exhausts the block limits", InvalidTransaction::Payment => "Inability to pay some fees (e.g. account balance too low)", + InvalidTransaction::BadMandatory => + "A call was labelled as mandatory, but resulted in an Error.", + InvalidTransaction::MandatoryDispatch => + "Tranaction dispatch is mandatory; transactions may not have mandatory dispatches.", InvalidTransaction::Custom(_) => "InvalidTransaction custom error", } } @@ -123,6 +142,15 @@ impl TransactionValidityError { Self::Unknown(_) => false, } } + + /// Returns `true` if the reason for the error was it being a mandatory dispatch that could not + /// be completed successfully. + pub fn was_mandatory(&self) -> bool { + match self { + Self::Invalid(e) => e.was_mandatory(), + Self::Unknown(_) => false, + } + } } impl From for &'static str { @@ -161,6 +189,35 @@ impl Into for UnknownTransaction { } } +/// The source of the transaction. +/// +/// Depending on the source we might apply different validation schemes. +/// For instance we can disallow specific kinds of transactions if they were not produced +/// by our local node (for instance off-chain workers). +#[derive(Copy, Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug, parity_util_mem::MallocSizeOf)] +pub enum TransactionSource { + /// Transaction is already included in block. + /// + /// This means that we can't really tell where the transaction is coming from, + /// since it's already in the received block. Note that the custom validation logic + /// using either `Local` or `External` should most likely just allow `InBlock` + /// transactions as well. + InBlock, + + /// Transaction is coming from a local source. + /// + /// This means that the transaction was produced internally by the node + /// (for instance an Off-Chain Worker, or an Off-Chain Call), as opposed + /// to being received over the network. + Local, + + /// Transaction has been received externally. + /// + /// This means the transaction has been received from (usually) "untrusted" source, + /// for instance received over the network or RPC. + External, +} + /// Information concerning a valid transaction. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] pub struct ValidTransaction { @@ -207,6 +264,17 @@ impl Default for ValidTransaction { } impl ValidTransaction { + /// Initiate `ValidTransaction` builder object with a particular prefix for tags. + /// + /// To avoid conflicts between different parts in runtime it's recommended to build `requires` + /// and `provides` tags with a unique prefix. + pub fn with_tag_prefix(prefix: &'static str) -> ValidTransactionBuilder { + ValidTransactionBuilder { + prefix: Some(prefix), + validity: Default::default(), + } + } + /// Combine two instances into one, as a best effort. This will take the superset of each of the /// `provides` and `requires` tags, it will sum the priorities, take the minimum longevity and /// the logic *And* of the propagate flags. @@ -221,6 +289,104 @@ impl ValidTransaction { } } +/// `ValidTransaction` builder. +/// +/// +/// Allows to easily construct `ValidTransaction` and most importantly takes care of +/// prefixing `requires` and `provides` tags to avoid conflicts. +#[derive(Default, Clone, RuntimeDebug)] +pub struct ValidTransactionBuilder { + prefix: Option<&'static str>, + validity: ValidTransaction, +} + +impl ValidTransactionBuilder { + /// Set the priority of a transaction. + /// + /// Note that the final priority for `FRAME` is combined from all `SignedExtension`s. + /// Most likely for unsigned transactions you want the priority to be higher + /// than for regular transactions. We recommend exposing a base priority for unsigned + /// transactions as a runtime module parameter, so that the runtime can tune inter-module + /// priorities. + pub fn priority(mut self, priority: TransactionPriority) -> Self { + self.validity.priority = priority; + self + } + + /// Set the longevity of a transaction. + /// + /// By default the transaction will be considered valid forever and will not be revalidated + /// by the transaction pool. It's recommended though to set the longevity to a finite value + /// though. If unsure, it's also reasonable to expose this parameter via module configuration + /// and let the runtime decide. + pub fn longevity(mut self, longevity: TransactionLongevity) -> Self { + self.validity.longevity = longevity; + self + } + + /// Set the propagate flag. + /// + /// Set to `false` if the transaction is not meant to be gossiped to peers. Combined with + /// `TransactionSource::Local` validation it can be used to have special kind of + /// transactions that are only produced and included by the validator nodes. + pub fn propagate(mut self, propagate: bool) -> Self { + self.validity.propagate = propagate; + self + } + + /// Add a `TransactionTag` to the set of required tags. + /// + /// The tag will be encoded and prefixed with module prefix (if any). + /// If you'd rather add a raw `require` tag, consider using `#combine_with` method. + pub fn and_requires(mut self, tag: impl Encode) -> Self { + self.validity.requires.push(match self.prefix.as_ref() { + Some(prefix) => (prefix, tag).encode(), + None => tag.encode(), + }); + self + } + + /// Add a `TransactionTag` to the set of provided tags. + /// + /// The tag will be encoded and prefixed with module prefix (if any). + /// If you'd rather add a raw `require` tag, consider using `#combine_with` method. + pub fn and_provides(mut self, tag: impl Encode) -> Self { + self.validity.provides.push(match self.prefix.as_ref() { + Some(prefix) => (prefix, tag).encode(), + None => tag.encode(), + }); + self + } + + /// Augment the builder with existing `ValidTransaction`. + /// + /// This method does add the prefix to `require` or `provides` tags. + pub fn combine_with(mut self, validity: ValidTransaction) -> Self { + self.validity = core::mem::take(&mut self.validity).combine_with(validity); + self + } + + /// Finalize the builder and produce `TransactionValidity`. + /// + /// Note the result will always be `Ok`. Use `Into` to produce `ValidTransaction`. + pub fn build(self) -> TransactionValidity { + self.into() + } +} + +impl From for TransactionValidity { + fn from(builder: ValidTransactionBuilder) -> Self { + Ok(builder.into()) + } +} + +impl From for ValidTransaction { + fn from(builder: ValidTransactionBuilder) -> Self { + builder.validity + } +} + + #[cfg(test)] mod tests { use super::*; @@ -244,4 +410,26 @@ mod tests { // decode back assert_eq!(TransactionValidity::decode(&mut &*encoded), Ok(v)); } + + #[test] + fn builder_should_prefix_the_tags() { + const PREFIX: &str = "test"; + let a: ValidTransaction = ValidTransaction::with_tag_prefix(PREFIX) + .and_requires(1) + .and_requires(2) + .and_provides(3) + .and_provides(4) + .propagate(false) + .longevity(5) + .priority(3) + .priority(6) + .into(); + assert_eq!(a, ValidTransaction { + propagate: false, + longevity: 5, + priority: 6, + requires: vec![(PREFIX, 1).encode(), (PREFIX, 2).encode()], + provides: vec![(PREFIX, 3).encode(), (PREFIX, 4).encode()], + }); + } } diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml index 3b9408dc8e5e39c280cfea35791f36314248ee28..5c1595027f86fd129ebec97ffebe63b2b1263ca7 100755 --- a/primitives/sandbox/Cargo.toml +++ b/primitives/sandbox/Cargo.toml @@ -14,7 +14,7 @@ sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../io" } sp-wasm-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../wasm-interface" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } [dev-dependencies] wabt = "0.9.2" @@ -31,3 +31,6 @@ std = [ "sp-wasm-interface/std", ] strict = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/serializer/Cargo.toml b/primitives/serializer/Cargo.toml index 355fe596de14d1fbf9a511c137cb5ee7dc12a2c5..75263321b805c85793e09f90ec0616ab1a6c4203 100644 --- a/primitives/serializer/Cargo.toml +++ b/primitives/serializer/Cargo.toml @@ -12,3 +12,6 @@ documentation = "https://docs.rs/sp-serializer" [dependencies] serde = "1.0.101" serde_json = "1.0.41" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index 0a10107f9857453a27c521c554fb1e7b5eaf78a0..ffe1bc327f7ffd9edb5f3c003abeb0f4d66077f3 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -17,3 +17,6 @@ sp-runtime = { version = "2.0.0-alpha.5", optional = true, path = "../runtime" } [features] default = [ "std" ] std = [ "sp-api/std", "sp-std/std", "sp-runtime", "sp-core/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/staking/Cargo.toml b/primitives/staking/Cargo.toml index 01f68ac3ce0611d37687f06f33bd68e14dea3800..60bf3f759e8dfcf0f3d2efe6bec7a3d231d291b8 100644 --- a/primitives/staking/Cargo.toml +++ b/primitives/staking/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "A crate which contains primitives that are useful for implementation that uses staking approaches in general. Definitions related to sessions, slashing, etc go here." [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } @@ -20,3 +20,6 @@ std = [ "sp-runtime/std", "sp-std/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/staking/src/offence.rs b/primitives/staking/src/offence.rs index 06e73f018b7656a691801b93b2178631cacd5ddd..584f3a75ea3abc2befd80821dba2f6b7349b2c5d 100644 --- a/primitives/staking/src/offence.rs +++ b/primitives/staking/src/offence.rs @@ -142,11 +142,20 @@ pub trait OnOffenceHandler { /// Zero is a valid value for a fraction. /// /// The `session` parameter is the session index of the offence. + /// + /// The receiver might decide to not accept this offence. In this case, the call site is + /// responsible for queuing the report and re-submitting again. fn on_offence( offenders: &[OffenceDetails], slash_fraction: &[Perbill], session: SessionIndex, - ); + ) -> Result<(), ()>; + + /// Can an offence be reported now or not. This is an method to short-circuit a call into + /// `on_offence`. Ideally, a correct implementation should return `false` if `on_offence` will + /// return `Err`. Nonetheless, this is up to the implementation and this trait cannot guarantee + /// it. + fn can_report() -> bool; } impl OnOffenceHandler for () { @@ -154,7 +163,9 @@ impl OnOffenceHandler for () { _offenders: &[OffenceDetails], _slash_fraction: &[Perbill], _session: SessionIndex, - ) {} + ) -> Result<(), ()> { Ok(()) } + + fn can_report() -> bool { true } } /// A details about an offending authority for a particular kind of offence. diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 47321181c77b8bbc7ebf491e3a96b241c5c38a94..548f3f43086e928986dd5c546f105e86e2ebf0c1 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -18,7 +18,7 @@ trie-root = "0.16.0" sp-trie = { version = "2.0.0-alpha.5", path = "../trie" } sp-core = { version = "2.0.0-alpha.5", path = "../core" } sp-panic-handler = { version = "2.0.0-alpha.5", path = "../panic-handler" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } num-traits = "0.2.8" rand = "0.7.2" sp-externalities = { version = "0.8.0-alpha.5", path = "../externalities" } @@ -29,3 +29,6 @@ sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } [features] default = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 4fb59556e3fe46f582fc9435b9b8b6a4006cb67f..94144fdb90fda5e04746e584993a678698b8e5f6 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -206,13 +206,16 @@ pub trait Backend: std::fmt::Debug { (root, txs) } + /// Register stats from overlay of state machine. + /// + /// By default nothing is registered. + fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats); + /// Query backend usage statistics (i/o, memory) /// /// Not all implementations are expected to be able to do this. In the /// case when they don't, empty statistics is returned. - fn usage_info(&self) -> UsageInfo { - UsageInfo::empty() - } + fn usage_info(&self) -> UsageInfo; /// Wipe the state database. fn wipe(&self) -> Result<(), Self::Error> { @@ -308,10 +311,12 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { (*self).for_key_values_with_prefix(prefix, f); } + fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } + fn usage_info(&self) -> UsageInfo { (*self).usage_info() } - } +} /// Trait that allows consolidate two transactions together. pub trait Consolidate { diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index c731d4104b260ef1680d6f800c9812544c03ab38..39ad81ed59a0eb3dc6746f922be36f334299801a 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -469,6 +469,7 @@ mod test { ].into_iter().collect(), }, collect_extrinsics: true, + stats: Default::default(), }; let config = Configuration { digest_interval: 4, digest_levels: 2 }; diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index b29446de2e5a42a684c471ab1e5740e7d53997ae..133af7ccd9b5f1234b4b9e8a823ea56e43953d8a 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -587,6 +587,7 @@ mod tests { ].into_iter().collect(), committed: Default::default(), collect_extrinsics: true, + stats: Default::default(), } } diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 7e474d45b650e480f35ee82bcae34828878a2ab3..ecd4532cf2640d5e590e6f7447cb84ec62e01bd0 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -20,6 +20,7 @@ use crate::{ StorageKey, StorageValue, StorageCollection, trie_backend::TrieBackend, backend::{Backend, insert_into_memory_db}, + stats::UsageInfo, }; use std::{error, fmt, collections::{BTreeMap, HashMap}, marker::PhantomData, ops}; use hash_db::Hasher; @@ -357,6 +358,16 @@ impl Backend for InMemory where H::Out: Codec { self.trie = Some(TrieBackend::new(mdb, root)); self.trie.as_ref() } + + fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } + + fn usage_info(&self) -> UsageInfo { + UsageInfo::empty() + } + + fn wipe(&self) -> Result<(), Self::Error> { + Ok(()) + } } #[cfg(test)] diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 2d992e778e67ef3be674668ca0d4408d62289e48..9a2dc52cca20a91ef3b545f0a7eec3ad2220e6a0 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -18,7 +18,7 @@ #![warn(missing_docs)] -use std::{fmt, result, collections::HashMap, panic::UnwindSafe, marker::PhantomData}; +use std::{fmt, result, collections::HashMap, panic::UnwindSafe}; use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode, Codec}; @@ -73,7 +73,7 @@ pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; pub use error::{Error, ExecutionError}; pub use in_memory_backend::InMemory as InMemoryBackend; -pub use stats::{UsageInfo, UsageUnit}; +pub use stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use sp_core::traits::CloneableSpawn; type CallResult = Result, E>; @@ -189,9 +189,19 @@ pub struct StateMachine<'a, B, H, N, Exec> overlay: &'a mut OverlayedChanges, extensions: Extensions, changes_trie_state: Option>, - _marker: PhantomData<(H, N)>, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, runtime_code: &'a RuntimeCode<'a>, + stats: StateMachineStats, +} + +impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> where + H: Hasher, + B: Backend, + N: ChangesTrieBlockNumber, +{ + fn drop(&mut self) { + self.backend.register_overlay_stats(&self.stats); + } } impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where @@ -224,9 +234,9 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where extensions, overlay, changes_trie_state, - _marker: PhantomData, storage_transaction_cache: None, runtime_code, + stats: StateMachineStats::default(), } } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 4eb44de7c58624a56c14ecd932edab5ca7ed3c74..ab50c61391bdb9f696bba1db1ca4d7874e9807a4 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -22,6 +22,7 @@ use crate::{ NO_EXTRINSIC_INDEX, BlockNumber, build_changes_trie, State as ChangesTrieState, }, + stats::StateMachineStats, }; #[cfg(test)] @@ -57,6 +58,8 @@ pub struct OverlayedChanges { pub(crate) committed: OverlayedChangeSet, /// True if extrinsics stats must be collected. pub(crate) collect_extrinsics: bool, + /// Collect statistic on this execution. + pub(crate) stats: StateMachineStats, } /// The storage value, used inside OverlayedChanges. @@ -206,7 +209,11 @@ impl OverlayedChanges { pub fn storage(&self, key: &[u8]) -> Option> { self.prospective.top.get(key) .or_else(|| self.committed.top.get(key)) - .map(|x| x.value.as_ref().map(AsRef::as_ref)) + .map(|x| { + let size_read = x.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_read_modified(size_read); + x.value.as_ref().map(AsRef::as_ref) + }) } /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred @@ -215,12 +222,16 @@ impl OverlayedChanges { pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.prospective.children.get(storage_key) { if let Some(val) = map.0.get(key) { + let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_read_modified(size_read); return Some(val.value.as_ref().map(AsRef::as_ref)); } } if let Some(map) = self.committed.children.get(storage_key) { if let Some(val) = map.0.get(key) { + let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_read_modified(size_read); return Some(val.value.as_ref().map(AsRef::as_ref)); } } @@ -232,6 +243,8 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_storage(&mut self, key: StorageKey, val: Option) { + let size_write = val.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_write_overlay(size_write); let extrinsic_index = self.extrinsic_index(); let entry = self.prospective.top.entry(key).or_default(); entry.value = val; @@ -252,6 +265,8 @@ impl OverlayedChanges { key: StorageKey, val: Option, ) { + let size_write = val.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_write_overlay(size_write); let extrinsic_index = self.extrinsic_index(); let map_entry = self.prospective.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 119fb59a7234e919765887bd859b04587cbb3292..747872af831ac44fbebaccf0c6c571b735b4f73a 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -283,6 +283,12 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> { self.0.child_storage_root(storage_key, child_info, delta) } + + fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } + + fn usage_info(&self) -> crate::stats::UsageInfo { + self.0.usage_info() + } } /// Create proof check backend. diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index aa69b5be9d652846f2c78ae1638bbe45aa08b877..8fa03344ad330af985fd52622f061c7b6644597f 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -17,6 +17,7 @@ //! Usage statistics for state db use std::time::{Instant, Duration}; +use std::cell::RefCell; /// Measured count of operations and total bytes. #[derive(Clone, Debug, Default)] @@ -32,10 +33,19 @@ pub struct UsageUnit { pub struct UsageInfo { /// Read statistics (total). pub reads: UsageUnit, - /// Write statistics. + /// Write statistics (total). pub writes: UsageUnit, + /// Write trie nodes statistics. + pub nodes_writes: UsageUnit, + /// Write into cached state machine + /// change overlay. + pub overlay_writes: UsageUnit, + /// Removed trie nodes statistics. + pub removed_nodes: UsageUnit, /// Cache read statistics. pub cache_reads: UsageUnit, + /// Modified value read statistics. + pub modified_reads: UsageUnit, /// Memory used. pub memory: usize, @@ -45,6 +55,35 @@ pub struct UsageInfo { pub span: Duration, } +/// Accumulated usage statistics specific to state machine +/// crate. +#[derive(Debug, Default, Clone)] +pub struct StateMachineStats { + /// Number of read query from runtime + /// that hit a modified value (in state + /// machine overlay). + pub reads_modified: RefCell, + /// Size in byte of read queries that + /// hit a modified value. + pub bytes_read_modified: RefCell, + /// Number of time a write operation + /// occurs into the state machine overlay. + pub writes_overlay: RefCell, + /// Size in bytes of the writes overlay + /// operation. + pub bytes_writes_overlay: RefCell, +} + +impl StateMachineStats { + /// Accumulates some registered stats. + pub fn add(&self, other: &StateMachineStats) { + *self.reads_modified.borrow_mut() += *other.reads_modified.borrow(); + *self.bytes_read_modified.borrow_mut() += *other.bytes_read_modified.borrow(); + *self.writes_overlay.borrow_mut() += *other.writes_overlay.borrow(); + *self.bytes_writes_overlay.borrow_mut() += *other.bytes_writes_overlay.borrow(); + } +} + impl UsageInfo { /// Empty statistics. /// @@ -53,10 +92,34 @@ impl UsageInfo { Self { reads: UsageUnit::default(), writes: UsageUnit::default(), + overlay_writes: UsageUnit::default(), + nodes_writes: UsageUnit::default(), + removed_nodes: UsageUnit::default(), cache_reads: UsageUnit::default(), + modified_reads: UsageUnit::default(), memory: 0, started: Instant::now(), span: Default::default(), } } + /// Add collected state machine to this state. + pub fn include_state_machine_states(&mut self, count: &StateMachineStats) { + self.modified_reads.ops += *count.reads_modified.borrow(); + self.modified_reads.bytes += *count.bytes_read_modified.borrow(); + self.overlay_writes.ops += *count.writes_overlay.borrow(); + self.overlay_writes.bytes += *count.bytes_writes_overlay.borrow(); + } +} + +impl StateMachineStats { + /// Tally one read modified operation, of some length. + pub fn tally_read_modified(&self, data_bytes: u64) { + *self.reads_modified.borrow_mut() += 1; + *self.bytes_read_modified.borrow_mut() += data_bytes; + } + /// Tally one write overlay operation, of some length. + pub fn tally_write_overlay(&self, data_bytes: u64) { + *self.writes_overlay.borrow_mut() += 1; + *self.bytes_writes_overlay.borrow_mut() += data_bytes; + } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e64dd590e51f711f2e59f0224f66e980121ee35f..f88e306a2fc3dad8ee1b7cfa6e45ead0f7e1a8b9 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -240,6 +240,12 @@ impl, H: Hasher> Backend for TrieBackend where fn as_trie_backend(&mut self) -> Option<&TrieBackend> { Some(self) } + + fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } + + fn usage_info(&self) -> crate::UsageInfo { + crate::UsageInfo::empty() + } } #[cfg(test)] diff --git a/primitives/std/Cargo.toml b/primitives/std/Cargo.toml index e33b9d88aa6f660ec3cf87b561d4a7dbbdd7f58e..58ff78f2bbb510adc98a7de88415503cf71b0061 100644 --- a/primitives/std/Cargo.toml +++ b/primitives/std/Cargo.toml @@ -13,3 +13,6 @@ documentation = "https://docs.rs/sp-std" [features] default = ["std"] std = [] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/storage/Cargo.toml b/primitives/storage/Cargo.toml index fa06f0cc2477bd27ba085a912437921d976cec89..e4e842848dd0fe002e613099352b9f731bfeeeba 100644 --- a/primitives/storage/Cargo.toml +++ b/primitives/storage/Cargo.toml @@ -18,3 +18,6 @@ sp-debug-derive = { version = "2.0.0-alpha.5", path = "../debug-derive" } [features] default = [ "std" ] std = [ "sp-std/std", "serde", "impl-serde" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/test-primitives/Cargo.toml b/primitives/test-primitives/Cargo.toml index 20f7d13dc96369d28e21b3758ff9cda314eef411..b8cb583835d89b4c25c0fa1583c36c8f66489729 100644 --- a/primitives/test-primitives/Cargo.toml +++ b/primitives/test-primitives/Cargo.toml @@ -10,7 +10,7 @@ publish = false [dependencies] sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } @@ -24,3 +24,6 @@ std = [ "sp-application-crypto/std", "serde", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/timestamp/Cargo.toml b/primitives/timestamp/Cargo.toml index cd31c1db25b84d30805103ec4120bf7668f6b35d..4a0851ccb19a843da2537b15d31843b0100e8cb3 100644 --- a/primitives/timestamp/Cargo.toml +++ b/primitives/timestamp/Cargo.toml @@ -12,7 +12,7 @@ description = "Substrate core types and inherents for timestamps." sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } impl-trait-for-tuples = "0.1.3" wasm-timer = "0.2" @@ -26,3 +26,6 @@ std = [ "codec/std", "sp-inherents/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/timestamp/src/lib.rs b/primitives/timestamp/src/lib.rs index 979b98b4952c92eaf3ad9b7f73d47807ada1d547..f1fd06a44a59f95e62d701f9b46fc4e4d0b9ce7e 100644 --- a/primitives/timestamp/src/lib.rs +++ b/primitives/timestamp/src/lib.rs @@ -77,6 +77,7 @@ impl TimestampInherentData for InherentData { } } +/// Provide duration since unix epoch in millisecond for timestamp inherent. #[cfg(feature = "std")] pub struct InherentDataProvider; diff --git a/primitives/transaction-pool/Cargo.toml b/primitives/transaction-pool/Cargo.toml index e345b1a1f0e18ce32d83caa3f3c71f855153a197..6e30fb4ddc16eb167fa786d27f110e53ab5857ff 100644 --- a/primitives/transaction-pool/Cargo.toml +++ b/primitives/transaction-pool/Cargo.toml @@ -11,13 +11,14 @@ documentation = "https://docs.rs/sp-transaction-pool" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.0", optional = true } derive_more = { version = "0.99.2", optional = true } futures = { version = "0.3.1", optional = true } log = { version = "0.4.8", optional = true } serde = { version = "1.0.101", features = ["derive"], optional = true} sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-utils = { version = "2.0.0-alpha.5", default-features = false, path = "../utils" } [features] default = [ "std" ] @@ -30,3 +31,6 @@ std = [ "sp-api/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/transaction-pool/src/lib.rs b/primitives/transaction-pool/src/lib.rs index 9abbcfbdf28aaf4a066f8e28fc4fb174d129939c..e4498bd0248801dbe7b2f57cb5917003d5932c62 100644 --- a/primitives/transaction-pool/src/lib.rs +++ b/primitives/transaction-pool/src/lib.rs @@ -29,5 +29,5 @@ mod pool; pub use pool::*; pub use sp_runtime::transaction_validity::{ - TransactionLongevity, TransactionPriority, TransactionTag, + TransactionLongevity, TransactionPriority, TransactionTag, TransactionSource, }; diff --git a/primitives/transaction-pool/src/pool.rs b/primitives/transaction-pool/src/pool.rs index cad06679647e4a7c90673b532d2c34c92b274125..ddc3fffa1530b4c40645b1b7a815284efb866bd9 100644 --- a/primitives/transaction-pool/src/pool.rs +++ b/primitives/transaction-pool/src/pool.rs @@ -22,16 +22,14 @@ use std::{ sync::Arc, pin::Pin, }; -use futures::{ - Future, Stream, - channel::mpsc, -}; +use futures::{Future, Stream,}; use serde::{Deserialize, Serialize}; +use sp_utils::mpsc; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Member, NumberFor}, transaction_validity::{ - TransactionLongevity, TransactionPriority, TransactionTag, + TransactionLongevity, TransactionPriority, TransactionTag, TransactionSource, }, }; @@ -132,7 +130,7 @@ pub enum TransactionStatus { pub type TransactionStatusStream = dyn Stream> + Send + Unpin; /// The import notification event stream. -pub type ImportNotificationStream = mpsc::UnboundedReceiver; +pub type ImportNotificationStream = mpsc::TracingUnboundedReceiver; /// Transaction hash type for a pool. pub type TxHash

=

::Hash; @@ -192,6 +190,7 @@ pub trait TransactionPool: Send + Sync { fn submit_at( &self, at: &BlockId, + source: TransactionSource, xts: Vec>, ) -> PoolFuture, Self::Error>>, Self::Error>; @@ -199,6 +198,7 @@ pub trait TransactionPool: Send + Sync { fn submit_one( &self, at: &BlockId, + source: TransactionSource, xt: TransactionFor, ) -> PoolFuture, Self::Error>; @@ -206,6 +206,7 @@ pub trait TransactionPool: Send + Sync { fn submit_and_watch( &self, at: &BlockId, + source: TransactionSource, xt: TransactionFor, ) -> PoolFuture>, Self::Error>; @@ -299,7 +300,9 @@ impl OffchainSubmitTransaction for TPool { extrinsic ); - let result = futures::executor::block_on(self.submit_one(&at, extrinsic)); + let result = futures::executor::block_on(self.submit_one( + &at, TransactionSource::Local, extrinsic, + )); result.map(|_| ()) .map_err(|e| log::warn!( diff --git a/primitives/transaction-pool/src/runtime_api.rs b/primitives/transaction-pool/src/runtime_api.rs index e30ce7c8289da7b00e33dac71e592c496a3dafb7..fa2e51653b284fcde5ca6362017c4ec813c48a0a 100644 --- a/primitives/transaction-pool/src/runtime_api.rs +++ b/primitives/transaction-pool/src/runtime_api.rs @@ -16,13 +16,27 @@ //! Tagged Transaction Queue Runtime API. -use sp_runtime::transaction_validity::TransactionValidity; +use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource}; use sp_runtime::traits::Block as BlockT; sp_api::decl_runtime_apis! { /// The `TaggedTransactionQueue` api trait for interfering with the transaction queue. + #[api_version(2)] pub trait TaggedTransactionQueue { - /// Validate the given transaction. + /// Validate the transaction. + #[changed_in(2)] fn validate_transaction(tx: ::Extrinsic) -> TransactionValidity; + + /// Validate the transaction. + /// + /// This method is invoked by the transaction pool to learn details about given transaction. + /// The implementation should make sure to verify the correctness of the transaction + /// against current state. + /// Note that this call may be performed by the pool multiple times and transactions + /// might be verified in any possible order. + fn validate_transaction( + source: TransactionSource, + tx: ::Extrinsic, + ) -> TransactionValidity; } } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index cb91377f96c1ab2bddcb23323cce6dbad9f1b7b8..3876f4652624be83193661e83ea74f85c108920a 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -14,7 +14,7 @@ name = "bench" harness = false [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } trie-db = { version = "0.20.0", default-features = false } @@ -40,3 +40,6 @@ std = [ "trie-root/std", "sp-core/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/utils/Cargo.toml b/primitives/utils/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..97e5ce1d9b618c2aa250ffd7054f4dbbc6165f14 --- /dev/null +++ b/primitives/utils/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sp-utils" +version = "2.0.0-alpha.5" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "I/O for Substrate runtimes" + +[dependencies] +futures = "0.3.4" +futures-core = "0.3.4" +lazy_static = "1.4.0" +prometheus = "0.8.0" + +[features] +default = ["metered"] +metered = [] diff --git a/primitives/utils/src/lib.rs b/primitives/utils/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..385a9b6689896102d5e43fc0eda53bb170f0443c --- /dev/null +++ b/primitives/utils/src/lib.rs @@ -0,0 +1,20 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Utilities Primitives for Substrate + +pub mod metrics; +pub mod mpsc; \ No newline at end of file diff --git a/primitives/utils/src/metrics.rs b/primitives/utils/src/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..160714fdca1cd8a6bc2864a38fead5bf7fa4cf56 --- /dev/null +++ b/primitives/utils/src/metrics.rs @@ -0,0 +1,58 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Metering primitives and globals + +use lazy_static::lazy_static; +use prometheus::{ + Registry, Error as PrometheusError, + core::{ AtomicU64, GenericGauge, GenericCounter }, +}; + +#[cfg(features = "metered")] +use prometheus::{core::GenericGaugeVec, Opts}; + + +lazy_static! { + pub static ref TOKIO_THREADS_TOTAL: GenericCounter = GenericCounter::new( + "tokio_threads_total", "Total number of threads created" + ).expect("Creating of statics doesn't fail. qed"); + + pub static ref TOKIO_THREADS_ALIVE: GenericGauge = GenericGauge::new( + "tokio_threads_alive", "Number of threads alive right now" + ).expect("Creating of statics doesn't fail. qed"); +} + +#[cfg(features = "metered")] +lazy_static! { + pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericGaugeVec = GenericGaugeVec::new( + Opts::new("unbounded_channel_len", "Items in each mpsc::unbounded instance"), + &["entity", "action"] // 'name of channel, send|received|dropped + ).expect("Creating of statics doesn't fail. qed"); + +} + + +/// Register the statics to report to registry +pub fn register_globals(registry: &Registry) -> Result<(), PrometheusError> { + registry.register(Box::new(TOKIO_THREADS_ALIVE.clone()))?; + registry.register(Box::new(TOKIO_THREADS_TOTAL.clone()))?; + + #[cfg(features = "metered")] + registry.register(Box::new(UNBOUNDED_CHANNELS_COUNTER.clone()))?; + + Ok(()) +} diff --git a/primitives/utils/src/mpsc.rs b/primitives/utils/src/mpsc.rs new file mode 100644 index 0000000000000000000000000000000000000000..42fb287c18d892ea204b60131dbea20c9f7ee43c --- /dev/null +++ b/primitives/utils/src/mpsc.rs @@ -0,0 +1,232 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Features to meter unbounded channels + +#[cfg(not(features = "metered"))] +mod inner { + // just aliased, non performance implications + use futures::channel::mpsc::{self, UnboundedReceiver, UnboundedSender}; + pub type TracingUnboundedSender = UnboundedSender; + pub type TracingUnboundedReceiver = UnboundedReceiver; + + /// Alias `mpsc::unbounded` + pub fn tracing_unbounded(_key: &'static str) ->(TracingUnboundedSender, TracingUnboundedReceiver) { + mpsc::unbounded() + } +} + + +#[cfg(features = "metered")] +mod inner { + //tracing implementation + use futures::channel::mpsc::{self, + UnboundedReceiver, UnboundedSender, + TryRecvError, TrySendError, SendError + }; + use futures::{sink::Sink, task::{Poll, Context}, stream::Stream}; + use std::pin::Pin; + use crate::metrics::UNBOUNDED_CHANNELS_COUNTER; + + /// Wrapper Type around `UnboundedSender` that increases the global + /// measure when a message is added + #[derive(Debug, Clone)] + pub struct TracingUnboundedSender(&'static str, UnboundedSender); + + /// Wrapper Type around `UnboundedReceiver` that decreases the global + /// measure when a message is polled + #[derive(Debug)] + pub struct TracingUnboundedReceiver(&'static str, UnboundedReceiver); + + /// Wrapper around `mpsc::unbounded` that tracks the in- and outflow via + /// `UNBOUNDED_CHANNELS_COUNTER` + pub fn tracing_unbounded(key: &'static str) ->(TracingUnboundedSender, TracingUnboundedReceiver) { + let (s, r) = mpsc::unbounded(); + (TracingUnboundedSender(key.clone(), s), TracingUnboundedReceiver(key,r)) + } + + impl TracingUnboundedSender { + /// Proxy function to mpsc::UnboundedSender + pub fn poll_ready(&self, ctx: &mut Context) -> Poll> { + self.1.poll_ready(ctx) + } + + /// Proxy function to mpsc::UnboundedSender + pub fn is_closed(&self) -> bool { + self.1.is_closed() + } + + /// Proxy function to mpsc::UnboundedSender + pub fn close_channel(&self) { + self.1.close_channel() + } + + /// Proxy function to mpsc::UnboundedSender + pub fn disconnect(&mut self) { + self.1.disconnect() + } + + /// Proxy function to mpsc::UnboundedSender + pub fn start_send(&mut self, msg: T) -> Result<(), SendError> { + self.1.start_send(msg) + } + + /// Proxy function to mpsc::UnboundedSender + pub fn unbounded_send(&self, msg: T) -> Result<(), TrySendError> { + self.1.unbounded_send(msg).map(|s|{ + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"send"]).incr(); + s + }) + } + + /// Proxy function to mpsc::UnboundedSender + pub fn same_receiver(&self, other: &UnboundedSender) -> bool { + self.1.same_receiver(other) + } + } + + impl TracingUnboundedReceiver { + + fn consume(&mut self) { + // consume all items, make sure to reflect the updated count + let mut count = 0; + while let Ok(Some(..)) = self.try_next() { + count += 1; + } + + // and discount the messages + if count > 0 { + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"dropped"]).incr_by(count); + } + + } + + /// Proxy function to mpsc::UnboundedReceiver + /// that consumes all messages first and updates the counter + pub fn close(&mut self) { + self.consume(); + self.1.close() + } + + /// Proxy function to mpsc::UnboundedReceiver + /// that discounts the messages taken out + pub fn try_next(&mut self) -> Result, TryRecvError> { + self.1.try_next().map(|s| { + if s.is_some() { + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"received"]).incr(); + } + s + }) + } + } + + impl Drop for TracingUnboundedReceiver { + fn drop(&mut self) { + self.consume(); + } + } + + impl Unpin for TracingUnboundedReceiver {} + + impl Stream for TracingUnboundedReceiver { + type Item = T; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let s = self.get_mut(); + match Pin::new(&mut s.1).poll_next(cx) { + Poll::Ready(msg) => { + if msg.is_some() { + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, "received"]).incr(); + } + Poll::Ready(msg) + } + Poll::Pending => { + Poll::Pending + } + } + } + } + + + impl Sink for TracingUnboundedSender { + type Error = SendError; + + fn poll_ready( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + TracingUnboundedSender::poll_ready(&*self, cx) + } + + fn start_send( + mut self: Pin<&mut Self>, + msg: T, + ) -> Result<(), Self::Error> { + TracingUnboundedSender::start_send(&mut *self, msg) + } + + fn poll_flush( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close( + mut self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll> { + self.disconnect(); + Poll::Ready(Ok(())) + } + } + + impl Sink for &TracingUnboundedSender { + type Error = SendError; + + fn poll_ready( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + TracingUnboundedSender::poll_ready(*self, cx) + } + + fn start_send(self: Pin<&mut Self>, msg: T) -> Result<(), Self::Error> { + self.unbounded_send(msg) + .map_err(TrySendError::into_send_error) + } + + fn poll_flush( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll> { + self.close_channel(); + Poll::Ready(Ok(())) + } + } +} + +pub use inner::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index 406eff4fbf1421d273fe57fa09a9617c872973f7..726d064642fb355a9d037ac840c44a5708db2850 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://docs.rs/sp-version" [dependencies] impl-serde = { version = "0.2.3", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } @@ -26,3 +26,6 @@ std = [ "sp-std/std", "sp-runtime/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/wasm-interface/Cargo.toml b/primitives/wasm-interface/Cargo.toml index 998463d6e9757a174fba7ba686e6659ea27be517..4a35d5b5180ff07529762e003370e7d2f21eef2b 100644 --- a/primitives/wasm-interface/Cargo.toml +++ b/primitives/wasm-interface/Cargo.toml @@ -13,8 +13,11 @@ documentation = "https://docs.rs/sp-wasm-interface" wasmi = { version = "0.6.2", optional = true } impl-trait-for-tuples = "0.1.2" sp-std = { version = "2.0.0-alpha.5", path = "../std", default-features = false } -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] std = [ "wasmi", "sp-std/std", "codec/std" ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 4584c3791035a7880271243727cf81c9d4c1b2f6..b8c9f9bd606059b73adb080f68c78dda3e8026a5 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -6,3 +6,6 @@ edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/client/Cargo.toml b/test-utils/client/Cargo.toml index 66fd7286968cd86a21948c6785b68783745c342a..ec87e7cd168f05585f9f70a25793fc4840af5017 100644 --- a/test-utils/client/Cargo.toml +++ b/test-utils/client/Cargo.toml @@ -14,11 +14,14 @@ sc-client = { version = "0.8.0-alpha.5", path = "../../client/" } sc-client-db = { version = "0.8.0-alpha.5", features = ["test-helpers"], path = "../../client/db" } sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } sc-executor = { version = "0.8.0-alpha.5", path = "../../client/executor" } -futures = "0.3.1" +futures = "0.3.4" hash-db = "0.15.2" sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 39920688e29c2832b3a4cfef7c980bc2f1e4fff5..be22747ea69166d91f83102e79171c18c8e5441a 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -15,7 +15,7 @@ sp-consensus-aura = { version = "0.8.0-alpha.5", default-features = false, path sp-consensus-babe = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/consensus/babe" } sp-block-builder = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/block-builder" } cfg-if = "0.1.10" -codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } frame-executive = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/executive" } sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" } @@ -87,3 +87,6 @@ std = [ "sp-transaction-pool/std", "trie-db/std", ] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/client/Cargo.toml b/test-utils/runtime/client/Cargo.toml index 3af6c4967eb0d74fb462c75d0ac067917b23f5d1..4be45fe46659a2583df6c02b2c67a32f8bf64a18 100644 --- a/test-utils/runtime/client/Cargo.toml +++ b/test-utils/runtime/client/Cargo.toml @@ -16,7 +16,10 @@ substrate-test-runtime = { version = "2.0.0-dev", path = "../../runtime" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } -futures = "0.3.1" +futures = "0.3.4" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 59955cce48693ea4a478378a404d116e32e2a791..c0aea9a2ab599ddc8a3366f10c7d6389aa83b18d 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -36,6 +36,7 @@ use sp_runtime::{ ApplyExtrinsicResult, create_runtime_str, Perbill, impl_opaque_keys, transaction_validity::{ TransactionValidity, ValidTransaction, TransactionValidityError, InvalidTransaction, + TransactionSource, }, traits::{ BlindCheckable, BlakeTwo256, Block as BlockT, Extrinsic as ExtrinsicT, @@ -180,6 +181,16 @@ impl ExtrinsicT for Extrinsic { } } +impl sp_runtime::traits::Dispatchable for Extrinsic { + type Origin = (); + type Trait = (); + type Info = (); + type PostInfo = (); + fn dispatch(self, _origin: Self::Origin) -> sp_runtime::DispatchResultWithInfo { + panic!("This implemention should not be used for actual dispatch."); + } +} + impl Extrinsic { pub fn transfer(&self) -> &Transfer { match self { @@ -492,7 +503,10 @@ cfg_if! { } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction(utx: ::Extrinsic) -> TransactionValidity { + fn validate_transaction( + _source: TransactionSource, + utx: ::Extrinsic, + ) -> TransactionValidity { if let Extrinsic::IncludeData(data) = utx { return Ok(ValidTransaction { priority: data.len() as u64, @@ -679,7 +693,10 @@ cfg_if! { } impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction(utx: ::Extrinsic) -> TransactionValidity { + fn validate_transaction( + _source: TransactionSource, + utx: ::Extrinsic, + ) -> TransactionValidity { if let Extrinsic::IncludeData(data) = utx { return Ok(ValidTransaction{ priority: data.len() as u64, diff --git a/test-utils/runtime/transaction-pool/Cargo.toml b/test-utils/runtime/transaction-pool/Cargo.toml index cc58bdcbd0c221c876afa1e3e821ac112f1fa316..52e2020dc8dd18dfc82de735e66b98b469bfefb9 100644 --- a/test-utils/runtime/transaction-pool/Cargo.toml +++ b/test-utils/runtime/transaction-pool/Cargo.toml @@ -11,10 +11,13 @@ publish = false [dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../client" } parking_lot = "0.10.0" -codec = { package = "parity-scale-codec", version = "1.2.0" } +codec = { package = "parity-scale-codec", version = "1.3.0" } sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } sc-transaction-graph = { version = "2.0.0-alpha.5", path = "../../../client/transaction-pool/graph" } futures = { version = "0.3.1", features = ["compat"] } derive_more = "0.99.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/transaction-pool/src/lib.rs b/test-utils/runtime/transaction-pool/src/lib.rs index 8cd4e58954b73c931f89b7178fee92896ce21c5e..432c9e520d1b178c0dc2567a35be82873d9ee687 100644 --- a/test-utils/runtime/transaction-pool/src/lib.rs +++ b/test-utils/runtime/transaction-pool/src/lib.rs @@ -25,6 +25,7 @@ use sp_runtime::{ traits::{BlakeTwo256, Hash as HashT}, transaction_validity::{ TransactionValidity, ValidTransaction, TransactionValidityError, InvalidTransaction, + TransactionSource, }, }; use std::collections::{HashSet, HashMap}; @@ -180,6 +181,7 @@ impl sc_transaction_graph::ChainApi for TestApi { fn validate_transaction( &self, _at: &BlockId, + _source: TransactionSource, uxt: sc_transaction_graph::ExtrinsicFor, ) -> Self::ValidationFuture { self.validation_requests.write().push(uxt.clone()); diff --git a/utils/browser/Cargo.toml b/utils/browser/Cargo.toml index 4e0a6ac7d158dcdd9097931e4523f41ff5db2da1..5f6b18e00148c5e34e694b1d86113bb873b88461 100644 --- a/utils/browser/Cargo.toml +++ b/utils/browser/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/paritytech/substrate/" futures = "0.3" futures01 = { package = "futures", version = "0.1.29" } log = "0.4.8" -libp2p-wasm-ext = { version = "0.16.2", features = ["websocket"] } +libp2p-wasm-ext = { version = "0.17.0", features = ["websocket"] } console_error_panic_hook = "0.1.6" console_log = "0.1.2" js-sys = "0.3.34" @@ -31,3 +31,6 @@ rand6 = { package = "rand", version = "0.6", features = ["wasm-bindgen"] } rand = { version = "0.7", features = ["wasm-bindgen"] } futures-timer = { version = "3.0.1", features = ["wasm-bindgen"]} chrono = { version = "0.4", features = ["wasmbind"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/browser/src/lib.rs b/utils/browser/src/lib.rs index 3a0162bd90abb5b0fdd8c09d500cafeae52dafd6..8bed06eee9621ead29f8200ae9d590ca0b423075 100644 --- a/utils/browser/src/lib.rs +++ b/utils/browser/src/lib.rs @@ -17,8 +17,10 @@ use futures01::sync::mpsc as mpsc01; use log::{debug, info}; use std::sync::Arc; +use sc_network::config::TransportConfig; use sc_service::{ - AbstractService, RpcSession, Roles, Configuration, config::{DatabaseConfig, KeystoreConfig}, + AbstractService, RpcSession, Role, Configuration, + config::{DatabaseConfig, KeystoreConfig, NetworkConfiguration}, GenericChainSpec, RuntimeGenesis }; use wasm_bindgen::prelude::*; @@ -43,29 +45,57 @@ where let name = chain_spec.name().to_string(); let transport = ExtTransport::new(ffi::websocket_transport()); - let mut config = Configuration::default(); - config.network.boot_nodes = chain_spec.boot_nodes().to_vec(); - config.telemetry_endpoints = chain_spec.telemetry_endpoints().clone(); - config.chain_spec = Some(Box::new(chain_spec)); - config.network.transport = sc_network::config::TransportConfig::Normal { + let mut network = NetworkConfiguration::new( + format!("{} (Browser)", name), + "unknown", + Default::default(), + &std::env::current_dir().expect("current directory must exist"), + ); + network.boot_nodes = chain_spec.boot_nodes().to_vec(); + network.transport = TransportConfig::Normal { wasm_external_transport: Some(transport.clone()), allow_private_ipv4: true, enable_mdns: false, use_yamux_flow_control: true, }; - config.task_executor = Some(Arc::new(move |fut| { - wasm_bindgen_futures::spawn_local(fut) - })); - config.telemetry_external_transport = Some(transport); - config.roles = Roles::LIGHT; - config.name = format!("{} (Browser)", name); - config.database = Some({ - info!("Opening Indexed DB database '{}'...", name); - let db = kvdb_web::Database::open(name, 10) - .await?; - DatabaseConfig::Custom(Arc::new(db)) - }); - config.keystore = KeystoreConfig::InMemory; + + let config = Configuration { + network, + telemetry_endpoints: chain_spec.telemetry_endpoints().clone(), + chain_spec: Box::new(chain_spec), + task_executor: Arc::new(move |fut| wasm_bindgen_futures::spawn_local(fut)), + telemetry_external_transport: Some(transport), + role: Role::Light, + database: { + info!("Opening Indexed DB database '{}'...", name); + let db = kvdb_web::Database::open(name, 10).await?; + + DatabaseConfig::Custom(Arc::new(db)) + }, + keystore: KeystoreConfig::InMemory, + default_heap_pages: Default::default(), + dev_key_seed: Default::default(), + disable_grandpa: Default::default(), + execution_strategies: Default::default(), + force_authoring: Default::default(), + impl_name: "parity-substrate", + impl_version: "0.0.0", + offchain_worker: Default::default(), + prometheus_config: Default::default(), + pruning: Default::default(), + rpc_cors: Default::default(), + rpc_http: Default::default(), + rpc_ws: Default::default(), + rpc_ws_max_connections: Default::default(), + state_cache_child_ratio: Default::default(), + state_cache_size: Default::default(), + tracing_receiver: Default::default(), + tracing_targets: Default::default(), + transaction_pool: Default::default(), + wasm_method: Default::default(), + max_runtime_instances: 8, + announce_block: true, + }; Ok(config) } diff --git a/utils/build-script-utils/Cargo.toml b/utils/build-script-utils/Cargo.toml index c5862a40eeabf12edec8dfee1588e5f3566380d3..637d50d19dde2c36265d920a5b3b545697549adc 100644 --- a/utils/build-script-utils/Cargo.toml +++ b/utils/build-script-utils/Cargo.toml @@ -9,3 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Crate with utility functions for `build.rs` scripts." [dependencies] +platforms = "0.2.1" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/build-script-utils/src/git.rs b/utils/build-script-utils/src/git.rs new file mode 100644 index 0000000000000000000000000000000000000000..919c4813abafaecb265a51b2b78183b1d5eb329c --- /dev/null +++ b/utils/build-script-utils/src/git.rs @@ -0,0 +1,123 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{env, fs, fs::File, io, io::Read, path::PathBuf}; + +/// Make sure the calling `build.rs` script is rerun when `.git/HEAD` changed. +/// +/// The file is searched from the `CARGO_MANIFEST_DIR` upwards. If the file can not be found, +/// a warning is generated. +pub fn rerun_if_git_head_changed() { + let mut manifest_dir = PathBuf::from( + env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."), + ); + let manifest_dir_copy = manifest_dir.clone(); + + while manifest_dir.parent().is_some() { + match get_git_paths(&manifest_dir) { + Err(err) => { + eprintln!("cargo:warning=Unable to read the Git repository: {}", err); + + return; + } + Ok(None) => {} + Ok(Some(paths)) => { + for p in paths { + println!("cargo:rerun-if-changed={}", p.display()); + } + + return; + } + } + + manifest_dir.pop(); + } + + println!( + "cargo:warning=Could not find `.git/HEAD` searching from `{}` upwards!", + manifest_dir_copy.display(), + ); +} + +// Code taken from https://github.com/rustyhorde/vergen/blob/8d522db8c8e16e26c0fc9ea8e6b0247cbf5cca84/src/output/envvar.rs +fn get_git_paths(path: &PathBuf) -> Result>, io::Error> { + let git_dir_or_file = path.join(".git"); + + if let Ok(metadata) = fs::metadata(&git_dir_or_file) { + if metadata.is_dir() { + // Echo the HEAD path + let git_head_path = git_dir_or_file.join("HEAD"); + + // Determine where HEAD points and echo that path also. + let mut f = File::open(&git_head_path)?; + let mut git_head_contents = String::new(); + let _ = f.read_to_string(&mut git_head_contents)?; + let ref_vec: Vec<&str> = git_head_contents.split(": ").collect(); + + if ref_vec.len() == 2 { + let current_head_file = ref_vec[1]; + let git_refs_path = PathBuf::from(".git").join(current_head_file); + + Ok(Some(vec![git_head_path, git_refs_path])) + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "You are most likely in a detached HEAD state", + )) + } + } else if metadata.is_file() { + // We are in a worktree, so find out where the actual worktrees//HEAD file is. + let mut git_file = File::open(&git_dir_or_file)?; + let mut git_contents = String::new(); + let _ = git_file.read_to_string(&mut git_contents)?; + let dir_vec: Vec<&str> = git_contents.split(": ").collect(); + let git_path = dir_vec[1].trim(); + + // Echo the HEAD psth + let git_head_path = PathBuf::from(git_path).join("HEAD"); + + // Find out what the full path to the .git dir is. + let mut actual_git_dir = PathBuf::from(git_path); + actual_git_dir.pop(); + actual_git_dir.pop(); + + // Determine where HEAD points and echo that path also. + let mut f = File::open(&git_head_path)?; + let mut git_head_contents = String::new(); + let _ = f.read_to_string(&mut git_head_contents)?; + let ref_vec: Vec<&str> = git_head_contents.split(": ").collect(); + + if ref_vec.len() == 2 { + let current_head_file = ref_vec[1]; + let git_refs_path = actual_git_dir.join(current_head_file); + + Ok(Some(vec![git_head_path, git_refs_path])) + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "You are most likely in a detached HEAD state", + )) + } + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "Invalid .git format (Not a directory or a file)", + )) + } + } else { + Ok(None) + } +} diff --git a/utils/build-script-utils/src/lib.rs b/utils/build-script-utils/src/lib.rs index 1b915bdcafacb1e5ecc0b81c60d062d0537ed496..57a1e7c5cdb3e8cae406b4e6d49b1dbc8cc27fb6 100644 --- a/utils/build-script-utils/src/lib.rs +++ b/utils/build-script-utils/src/lib.rs @@ -16,29 +16,8 @@ //! Crate with utility functions for `build.rs` scripts. -use std::{env, path::PathBuf}; +mod version; +mod git; -/// Make sure the calling `build.rs` script is rerun when `.git/HEAD` changed. -/// -/// The file is searched from the `CARGO_MANIFEST_DIR` upwards. If the file can not be found, -/// a warning is generated. -pub fn rerun_if_git_head_changed() { - let mut manifest_dir = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo.") - ); - let manifest_dir_copy = manifest_dir.clone(); - - while manifest_dir.parent().is_some() { - if manifest_dir.join(".git/HEAD").exists() { - println!("cargo:rerun-if-changed={}", manifest_dir.join(".git/HEAD").display()); - return - } - - manifest_dir.pop(); - } - - println!( - "cargo:warning=Could not find `.git/HEAD` searching from `{}` upwards!", - manifest_dir_copy.display(), - ); -} +pub use git::*; +pub use version::*; diff --git a/utils/build-script-utils/src/version.rs b/utils/build-script-utils/src/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..d41538707c27eb4ce08a3d6fa7d9cf7650f627c4 --- /dev/null +++ b/utils/build-script-utils/src/version.rs @@ -0,0 +1,59 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use platforms::*; +use std::process::Command; + +/// Generate the `cargo:` key output +pub fn generate_cargo_keys() { + let output = Command::new("git") + .args(&["rev-parse", "--short", "HEAD"]) + .output(); + + match output { + Ok(o) if o.status.success() => { + let sha = String::from_utf8_lossy(&o.stdout).trim().to_owned(); + + println!("cargo:rustc-env=SUBSTRATE_CLI_IMPL_VERSION={}", get_version(sha.as_str())) + } + Ok(o) => eprintln!("cargo:warning=Git command failed with status: {}", o.status), + Err(err) => eprintln!("cargo:warning=Failed to execute git command: {}", err), + } +} + +fn get_platform() -> String { + let env_dash = if TARGET_ENV.is_some() { "-" } else { "" }; + + format!( + "{}-{}{}{}", + TARGET_ARCH.as_str(), + TARGET_OS.as_str(), + env_dash, + TARGET_ENV.map(|x| x.as_str()).unwrap_or(""), + ) +} + +fn get_version(impl_commit: &str) -> String { + let commit_dash = if impl_commit.is_empty() { "" } else { "-" }; + + format!( + "{}{}{}-{}", + std::env::var("CARGO_PKG_VERSION").unwrap_or_default(), + commit_dash, + impl_commit, + get_platform(), + ) +} diff --git a/utils/fork-tree/Cargo.toml b/utils/fork-tree/Cargo.toml index 0814a761ecec2cd766cb7d79a2bb152e50ec4b57..e46618feb8e5a9d8dc4aed8f100c5f6bfcea1323 100644 --- a/utils/fork-tree/Cargo.toml +++ b/utils/fork-tree/Cargo.toml @@ -10,4 +10,7 @@ description = "Utility library for managing tree-like ordered data with logic fo documentation = "https://docs.rs/fork-tree" [dependencies] -codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index d63e516b45367fcd8cc12c98217d63176aa575b8..f4b72187478d47c63575c01ad27d86500c160816 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -20,8 +20,11 @@ sp-externalities = { version = "0.8.0-alpha.5", path = "../../../primitives/exte sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } structopt = "0.3.8" -codec = { version = "1.2.0", package = "parity-scale-codec" } +codec = { version = "1.3.0", package = "parity-scale-codec" } [features] default = ["rocksdb"] rocksdb = ["sc-client-db/kvdb-rocksdb"] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs new file mode 100644 index 0000000000000000000000000000000000000000..5e35d57cdaee9eb6b7523f4a17b38d1aa70d7b5b --- /dev/null +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -0,0 +1,141 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::BenchmarkCmd; +use codec::{Decode, Encode}; +use frame_benchmarking::{Analysis, BenchmarkBatch}; +use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result}; +use sc_client::StateMachine; +use sc_client_db::BenchmarkingState; +use sc_executor::NativeExecutor; +use sp_externalities::Extensions; +use sc_service::{Configuration, NativeExecutionDispatch}; +use sp_runtime::{ + traits::{Block as BlockT, Header as HeaderT, NumberFor}, +}; +use sp_core::{tasks, testing::KeyStore, traits::KeystoreExt}; +use std::fmt::Debug; + +impl BenchmarkCmd { + /// Runs the command and benchmarks the chain. + pub fn run(&self, config: Configuration) -> Result<()> + where + BB: BlockT + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + ExecDispatch: NativeExecutionDispatch + 'static, + { + let spec = config.chain_spec; + let wasm_method = self.wasm_method.into(); + let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); + + let genesis_storage = spec.build_storage()?; + let mut changes = Default::default(); + let cache_size = Some(self.database_cache_size as usize); + let state = BenchmarkingState::::new(genesis_storage, cache_size)?; + let executor = NativeExecutor::::new( + wasm_method, + None, // heap pages + 2, // The runtime instances cache size. + ); + + let mut extensions = Extensions::default(); + extensions.register(KeystoreExt(KeyStore::new())); + + let result = StateMachine::<_, _, NumberFor, _>::new( + &state, + None, + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + &self.pallet, + &self.extrinsic, + self.lowest_range_values.clone(), + self.highest_range_values.clone(), + self.steps.clone(), + self.repeat, + ).encode(), + extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, + tasks::executor(), + ) + .execute(strategy.into()) + .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; + + let results = , String> as Decode>::decode(&mut &result[..]) + .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; + + match results { + Ok(batches) => for batch in batches.into_iter() { + // Print benchmark metadata + println!( + "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", + String::from_utf8(batch.pallet).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), + self.lowest_range_values, + self.highest_range_values, + self.steps, + self.repeat, + ); + + if self.raw_data { + // Print the table header + batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + + print!("extrinsic_time,storage_root_time\n"); + // Print the values + batch.results.iter().for_each(|result| { + let parameters = &result.0; + parameters.iter().for_each(|param| print!("{:?},", param.1)); + // Print extrinsic time and storage root time + print!("{:?},{:?}\n", result.1, result.2); + }); + + println!(); + } + + // Conduct analysis. + if !self.no_median_slopes { + if let Some(analysis) = Analysis::median_slopes(&batch.results) { + println!("Median Slopes Analysis\n========\n{}", analysis); + } + } + if !self.no_min_squares { + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { + println!("Min Squares Analysis\n========\n{}", analysis); + } + } + }, + Err(error) => eprintln!("Error: {:?}", error), + } + + Ok(()) + } +} + +impl CliConfiguration for BenchmarkCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn chain_id(&self, _is_dev: bool) -> Result { + Ok(match self.shared_params.chain { + Some(ref chain) => chain.clone(), + None => "dev".into(), + }) + } +} diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 1c02a754016e3dd006f5f35cf3c79c3c44c70b30..96204d1ae576382feb3ac68ab9b5c26ed5a90d61 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -14,30 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +mod command; + +use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::fmt::Debug; -use sp_runtime::{traits::{Block as BlockT, Header as HeaderT, NumberFor}}; -use sc_client::StateMachine; -use sc_cli::{ExecutionStrategy, WasmExecutionMethod, VersionInfo}; -use sc_client_db::BenchmarkingState; -use sc_service::{Configuration, ChainSpec}; -use sc_executor::{NativeExecutor, NativeExecutionDispatch}; -use codec::{Encode, Decode}; -use frame_benchmarking::{BenchmarkResults, Analysis}; -use sp_core::{ - tasks, - traits::KeystoreExt, - testing::KeyStore, -}; -use sp_externalities::Extensions; /// The `benchmark` command used to benchmark FRAME Pallets. #[derive(Debug, structopt::StructOpt, Clone)] pub struct BenchmarkCmd { - /// Select a FRAME Pallet to benchmark. + /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). #[structopt(short, long)] pub pallet: String, - /// Select an extrinsic to benchmark. + /// Select an extrinsic inside the pallet to benchmark, or `*` for all. #[structopt(short, long)] pub extrinsic: String, @@ -46,17 +35,29 @@ pub struct BenchmarkCmd { pub steps: Vec, /// Indicates lowest values for each of the component ranges. - #[structopt(long, use_delimiter = true)] + #[structopt(long = "low", use_delimiter = true)] pub lowest_range_values: Vec, /// Indicates highest values for each of the component ranges. - #[structopt(long, use_delimiter = true)] + #[structopt(long = "high", use_delimiter = true)] pub highest_range_values: Vec, /// Select how many repetitions of this benchmark should run. #[structopt(short, long, default_value = "1")] pub repeat: u32, + /// Print the raw results. + #[structopt(long = "raw")] + pub raw_data: bool, + + /// Don't print the median-slopes linear regression analysis. + #[structopt(long)] + pub no_median_slopes: bool, + + /// Don't print the min-squares linear regression analysis. + #[structopt(long)] + pub no_min_squares: bool, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, @@ -79,125 +80,8 @@ pub struct BenchmarkCmd { default_value = "Interpreted" )] pub wasm_method: WasmExecutionMethod, -} -impl BenchmarkCmd { - /// Initialize - pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { - self.shared_params.init(version) - } - - /// Runs the command and benchmarks the chain. - pub fn run( - self, - config: Configuration, - ) -> sc_cli::Result<()> - where - BB: BlockT + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - ExecDispatch: NativeExecutionDispatch + 'static, - { - let spec = config.chain_spec.expect("chain_spec is always Some"); - let wasm_method = self.wasm_method.into(); - let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); - - let genesis_storage = spec.build_storage()?; - let mut changes = Default::default(); - let state = BenchmarkingState::::new(genesis_storage)?; - let executor = NativeExecutor::::new( - wasm_method, - None, // heap pages - 2, // The runtime instances cache size. - ); - - let mut extensions = Extensions::default(); - extensions.register(KeystoreExt(KeyStore::new())); - - let result = StateMachine::<_, _, NumberFor, _>::new( - &state, - None, - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &self.pallet, - &self.extrinsic, - self.lowest_range_values.clone(), - self.highest_range_values.clone(), - self.steps.clone(), - self.repeat, - ).encode(), - extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, - tasks::executor(), - ) - .execute(strategy.into()) - .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; - - let results = , String> as Decode>::decode(&mut &result[..]) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; - - match results { - Ok(results) => { - // Print benchmark metadata - println!( - "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - self.pallet, - self.extrinsic, - self.lowest_range_values, - self.highest_range_values, - self.steps, - self.repeat, - ); - - // Print the table header - results[0].0.iter().for_each(|param| print!("{:?},", param.0)); - - print!("extrinsic_time,storage_root_time\n"); - // Print the values - results.iter().for_each(|result| { - let parameters = &result.0; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.1, result.2); - }); - - print!("\n"); - - // Conduct analysis. - if let Some(analysis) = Analysis::median_slopes(&results) { - println!("Median Slopes Analysis\n========\n{}", analysis); - } - - if let Some(analysis) = Analysis::min_squares_iqr(&results) { - println!("Min Squares Analysis\n========\n{}", analysis); - } - - eprintln!("Done."); - } - Err(error) => eprintln!("Error: {:?}", error), - } - - Ok(()) - } - - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: impl FnOnce(&str) -> Result, String>, - _version: &VersionInfo, - ) -> sc_cli::Result<()> - { - // Configure chain spec. - let chain_key = self.shared_params.chain.clone().unwrap_or("dev".into()); - let spec = spec_factory(&chain_key)?; - config.chain_spec = Some(spec); - - // Make sure to configure keystore. - config.use_in_memory_keystore()?; - - Ok(()) - } + /// Limit the memory the database cache can use. + #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] + pub database_cache_size: u32, } diff --git a/utils/frame/rpc/support/Cargo.toml b/utils/frame/rpc/support/Cargo.toml index 4de85524e70dce8caac705e8d9b7f4c97c32be07..72884330d2e8224cc9959beb5c23bcd67166dea1 100644 --- a/utils/frame/rpc/support/Cargo.toml +++ b/utils/frame/rpc/support/Cargo.toml @@ -10,7 +10,7 @@ description = "Substrate RPC for FRAME's support" [dependencies] futures = { version = "0.3.0", features = ["compat"] } -jsonrpc-client-transports = "14" +jsonrpc-client-transports = { version = "14.0.5", default-features = false, features = ["http"] } jsonrpc-core = "14" codec = { package = "parity-scale-codec", version = "1" } serde = "1" @@ -21,3 +21,6 @@ sc-rpc-api = { version = "0.8.0-alpha.5", path = "../../../../client/rpc-api" } [dev-dependencies] frame-system = { version = "2.0.0-alpha.5", path = "../../../../frame/system" } tokio = "0.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml index a4f6833fad16d379ab9a868318750e10c551124b..a9e775393dfb17702933d17399a80824872ba5ee 100644 --- a/utils/frame/rpc/system/Cargo.toml +++ b/utils/frame/rpc/system/Cargo.toml @@ -10,10 +10,10 @@ description = "FRAME's system exposed over Substrate RPC" [dependencies] sc-client = { version = "0.8.0-alpha.5", path = "../../../../client/" } -codec = { package = "parity-scale-codec", version = "1.2.0" } -futures = "0.3.1" +codec = { package = "parity-scale-codec", version = "1.3.0" } +futures = "0.3.4" jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" +jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" log = "0.4.8" serde = { version = "1.0.101", features = ["derive"] } @@ -28,3 +28,6 @@ sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../../primitive substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../test-utils/runtime/client" } env_logger = "0.7.0" sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../../client/transaction-pool" } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index e6214c73d6ea1ab3f0a3b4ffc9cc3d05d4cdd660..c73ddfe93efa067d8188f9abb66c554c9865baad 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -239,6 +239,7 @@ mod tests { BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 ); + let source = sp_runtime::transaction_validity::TransactionSource::External; let new_transaction = |nonce: u64| { let t = Transfer { from: AccountKeyring::Alice.into(), @@ -250,9 +251,9 @@ mod tests { }; // Populate the pool let ext0 = new_transaction(0); - block_on(pool.submit_one(&BlockId::number(0), ext0)).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), source, ext0)).unwrap(); let ext1 = new_transaction(1); - block_on(pool.submit_one(&BlockId::number(0), ext1)).unwrap(); + block_on(pool.submit_one(&BlockId::number(0), source, ext1)).unwrap(); let accounts = FullSystem::new(client, pool); diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml index bad77b1ea63cbe66b52d2303b6984a82d4ddd71c..add4b0da5ff9565b76e0efb25c5c7ceb888e709a 100644 --- a/utils/prometheus/Cargo.toml +++ b/utils/prometheus/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/paritytech/substrate/" [dependencies] log = "0.4.8" -prometheus = "0.7" +prometheus = "0.8" futures-util = { version = "0.3.1", default-features = false, features = ["io"] } derive_more = "0.99" @@ -18,3 +18,6 @@ derive_more = "0.99" async-std = { version = "1.0.1", features = ["unstable"] } hyper = { version = "0.13.1", default-features = false, features = ["stream"] } tokio = "0.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/prometheus/src/lib.rs b/utils/prometheus/src/lib.rs index 54b9183bc63929859631e5aba7a02e87ac026e8a..9030704cb746ffbb4fa9ebfe5f468751a67a28d2 100644 --- a/utils/prometheus/src/lib.rs +++ b/utils/prometheus/src/lib.rs @@ -16,7 +16,10 @@ use futures_util::{FutureExt, future::Future}; pub use prometheus::{ + self, Registry, Error as PrometheusError, Opts, + Histogram, HistogramOpts, HistogramVec, + exponential_buckets, core::{ GenericGauge as Gauge, GenericCounter as Counter, GenericGaugeVec as GaugeVec, GenericCounterVec as CounterVec, @@ -120,7 +123,7 @@ mod known_os { .await .map_err(|_| Error::PortInUse(prometheus_addr))?; - log::info!("Prometheus server started at {}", prometheus_addr); + log::info!("〽️ Prometheus server started at {}", prometheus_addr); let service = make_service_fn(move |_| { let registry = registry.clone(); diff --git a/utils/wasm-builder-runner/Cargo.toml b/utils/wasm-builder-runner/Cargo.toml index 8a41fe98b2728efee0a0e71c946853f4adde7be2..77796ea8d9a8b15c21d7531fb772afc43c47ee6a 100644 --- a/utils/wasm-builder-runner/Cargo.toml +++ b/utils/wasm-builder-runner/Cargo.toml @@ -10,3 +10,6 @@ license = "GPL-3.0" homepage = "https://substrate.dev" [dependencies] + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index 1aac8913939b1253e66ad50e53dd18d3d1177cd8..ed953cca577ddf1deaec0f8600b7d473293ab24c 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -19,3 +19,6 @@ fs2 = "0.4.3" wasm-gc-api = "0.1.11" atty = "0.2.13" itertools = "0.8.2" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"]