diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62a6c2de32ddfa86fdbe6736737f7f0f3dc1768d..bc7ce84a80512e5fa7554bc1f13e82974340ed3a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,21 @@ # pipelines can be triggered manually in the web # setting DEPLOY_TAG will only deploy the tagged image +# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a +# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ +# +# my-example-job: +# stage: test # One of the stages listed below this job (required) +# image: parity/tools:latest # Any docker image (required) +# allow_failure: true # Allow the pipeline to continue if this job fails (default: false) +# dependencies: +# - build-rust-doc-release # Any jobs that are required to run before this job (optional) +# variables: +# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) +# script: +# - echo "List of shell commands to run in your job" +# - echo "You can also just specify a script here, like so:" +# - ./.maintain/gitlab/my_amazing_script.sh stages: - test @@ -22,6 +37,9 @@ variables: CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" ARCH: "x86_64" + # FIXME set to release + CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.6" + CARGO_UNLEASH_PKG_DEF: "--skip node node-* subkey chain-spec-builder" .collect-artifacts: &collect-artifacts @@ -86,6 +104,16 @@ check-runtime: interruptible: true allow_failure: true +check-signed-tag: + stage: test + image: parity/tools:latest + <<: *kubernetes-build + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - ./.maintain/gitlab/check_signed.sh + allow_failure: false check-line-width: stage: test @@ -160,6 +188,16 @@ test-dependency-rules: script: - .maintain/ensure-deps.sh +unleash-check: + stage: test + <<: *docker-env + only: + - master + - tags + script: + - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} + - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} + test-frame-staking: stage: test <<: *docker-env @@ -524,7 +562,28 @@ publish-gh-doc: after_script: - rm -vrf ${HOME}/.gitconfig +publish-draft-release: + stage: publish + image: parity/tools:latest + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - ./.maintain/gitlab/publish_draft_release.sh + interruptible: true + allow_failure: true +publish-to-crates-io: + stage: publish + <<: *docker-env + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} + - cargo unleash em-dragons --no-check ${CARGO_UNLEASH_PKG_DEF} + interruptible: true + allow_failure: true .deploy-template: &deploy stage: kubernetes diff --git a/.maintain/Dockerfile b/.maintain/Dockerfile index 7cba85c544afc2c8cc1ff56401b2172a01d30364..56bfc7a2cc48130b659ce7e0a5127d3119b59d12 100644 --- a/.maintain/Dockerfile +++ b/.maintain/Dockerfile @@ -1,7 +1,7 @@ # Note: We don't use Alpine and its packaged Rust/Cargo because they're too often out of date, # preventing them from being used to build Substrate/Polkadot. -FROM phusion/baseimage:0.10.2 as builder +FROM phusion/baseimage:0.11 as builder LABEL maintainer="chevdor@gmail.com" LABEL description="This is the build stage for Substrate. Here we create the binary." @@ -26,7 +26,7 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ # ===== SECOND STAGE ====== -FROM phusion/baseimage:0.10.2 +FROM phusion/baseimage:0.11 LABEL maintainer="chevdor@gmail.com" LABEL description="This is the 2nd stage: a very small image where we copy the Substrate binary." ARG PROFILE=release diff --git a/.maintain/gitlab/check_signed.sh b/.maintain/gitlab/check_signed.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c4cc47baba38fa41214ed0fefc7e09b75a69e7d --- /dev/null +++ b/.maintain/gitlab/check_signed.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +version="$CI_COMMIT_TAG" + +echo '[+] Checking tag has been signed' +check_tag "paritytech/substrate" "$version" +case $? in + 0) echo '[+] Tag found and has been signed'; exit 0 + ;; + 1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1 + ;; + 2) echo '[!] Tag not found. Aborting release.'; exit 1 +esac diff --git a/.maintain/gitlab/lib.sh b/.maintain/gitlab/lib.sh new file mode 100755 index 0000000000000000000000000000000000000000..bc0e06a6d46c4e7c407bef6998accab50586f366 --- /dev/null +++ b/.maintain/gitlab/lib.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +api_base="https://api.github.com/repos" + +# Function to take 2 git tags/commits and get any lines from commit messages +# that contain something that looks like a PR reference: e.g., (#1234) +sanitised_git_logs(){ + git --no-pager log --pretty=format:"%s" "$1..$2" | + # Only find messages referencing a PR + grep -E '\(#[0-9]+\)' | + # Strip any asterisks + sed 's/^* //g' | + # And add them all back + sed 's/^/* /g' +} + +# Checks whether a tag on github has been verified +# repo: 'organization/repo' +# tagver: 'v1.2.3' +# Usage: check_tag $repo $tagver +check_tag () { + repo=$1 + tagver=$2 + tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver") + tag_sha=$(echo "$tag_out" | jq -r .object.sha) + object_url=$(echo "$tag_out" | jq -r .object.url) + if [ "$tag_sha" = "null" ]; then + return 2 + fi + verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified) + if [ "$verified_str" = "true" ]; then + # Verified, everything is good + return 0 + else + # Not verified. Bad juju. + return 1 + fi +} + +# Checks whether a given PR has a given label. +# repo: 'organization/repo' +# pr_id: 12345 +# label: B1-silent +# Usage: has_label $repo $pr_id $label +has_label(){ + repo="$1" + pr_id="$2" + label="$3" + out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id") + [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ] +} + +# Formats a message into a JSON string for posting to Matrix +# message: 'any plaintext message' +# formatted_message: 'optional message formatted in html' +# Usage: structure_message $content $formatted_content (optional) +structure_message() { + if [ -z "$2" ]; then + body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null) + else + body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null) + fi + echo "$body" +} + +# Post a message to a matrix room +# body: '{body: "JSON string produced by structure_message"}' +# room_id: !fsfSRjgjBWEWffws:matrix.parity.io +# access_token: see https://matrix.org/docs/guides/client-server-api/ +# Usage: send_message $body (json formatted) $room_id $access_token +send_message() { +curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" +} diff --git a/.maintain/gitlab/publish_draft_release.sh b/.maintain/gitlab/publish_draft_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..8566827a0992087b9096f4ab1aa432ed24bb38da --- /dev/null +++ b/.maintain/gitlab/publish_draft_release.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +# Substrate labels for PRs we want to include in the release notes +labels=( + 'B1-runtimenoteworthy' + 'B1-clientnoteworthy' + 'B1-apinoteworthy' +) + +version="$CI_COMMIT_TAG" +last_version=$(git tag -l | sort -V | grep -B 1 -x "$version" | head -n 1) +echo "[+] Version: $version; Previous version: $last_version" + +all_changes="$(sanitised_git_logs "$last_version" "$version")" +labelled_changes="" +echo "[+] Iterating through $(wc -l <<< "$all_changes") changes to find labelled PRs" +while IFS= read -r line; do + pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/') + + # Skip if the PR has the silent label - this allows us to skip a few requests + if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then + continue + fi + for label in "${labels[@]}"; do + if has_label 'paritytech/substrate' "$pr_id" "$label"; then + labelled_changes="$labelled_changes +$line" + fi + done +done <<< "$all_changes" + + +release_text="Substrate $version +----------------- +$labelled_changes" + +echo "[+] Release text generated: " +echo "$release_text" +exit +echo "[+] Pushing release to github" +# Create release on github +release_name="Substrate $version" +data=$(jq -Rs --arg version "$version" \ + --arg release_name "$release_name" \ + --arg release_text "$release_text" \ +'{ + "tag_name": $version, + "target_commitish": "master", + "name": $release_name, + "body": $release_text, + "draft": true, + "prerelease": false +}' < /dev/null) + +out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases") + +html_url=$(echo "$out" | jq -r .html_url) + +if [ "$html_url" == "null" ] +then + echo "[!] Something went wrong posting:" + echo "$out" +else + echo "[+] Release draft created: $html_url" +fi + +echo '[+] Sending draft release URL to Matrix' + +msg_body=$(cat <Release pipeline for Substrate $version complete.
+Draft release created: $html_url +EOF +) +send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ACCESS_TOKEN" + +echo "[+] Done! Maybe the release worked..." diff --git a/.maintain/kubernetes/templates/service.yaml b/.maintain/kubernetes/templates/service.yaml index 01ba9d5a567c57045b585aa7e1fa894d404cbf90..b14bb74c10a1abdeae784a51189500e46bce131d 100644 --- a/.maintain/kubernetes/templates/service.yaml +++ b/.maintain/kubernetes/templates/service.yaml @@ -33,7 +33,7 @@ spec: app: {{ .Values.GitlabEnvSlug | default .Values.app }} sessionAffinity: None type: NodePort - # don't route exteral traffic to non-local pods + # don't route external traffic to non-local pods externalTrafficPolicy: Local {{- else if .Values.validator.keys }} {{- $root := . -}} diff --git a/.maintain/kubernetes/values.yaml b/.maintain/kubernetes/values.yaml index 89a6445e00a2f59c6290ef8e4eafdc9fdd925832..4c3cb5c7d702d6c4a8a3ce6b428012dcd634e3db 100644 --- a/.maintain/kubernetes/values.yaml +++ b/.maintain/kubernetes/values.yaml @@ -48,7 +48,7 @@ validator: {} # substrate-1-node-key # # pod names are canonical. changing these or providing different amount of - # keys than the replicas count will lead to behavior noone ever has + # keys than the replicas count will lead to behavior no one ever has # experienced before. diff --git a/Cargo.lock b/Cargo.lock index d13c45afa22f73383a0e2c9103d1ee7428ddc9a3..53e2a5663b09c5c0acb763289213c0b2b2692b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,7 +146,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -189,7 +189,7 @@ dependencies = [ "mio", "mio-uds", "num_cpus", - "once_cell 1.3.1", + "once_cell", "pin-project-lite", "pin-utils", "slab", @@ -242,9 +242,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" +checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" dependencies = [ "backtrace-sys", "cfg-if", @@ -305,25 +305,26 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.49.4" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e" +checksum = "99de13bb6361e01e493b3db7928085dcc474b7ba4f5481818e53a89d76b8393f" dependencies = [ "bitflags", "cexpr", "cfg-if", "clang-sys", "clap", - "env_logger 0.6.2", - "fxhash", + "env_logger 0.7.1", "lazy_static", + "lazycell", "log 0.4.8", "peeking_take_while", - "proc-macro2 0.4.30", - "quote 0.6.13", + "proc-macro2", + "quote", "regex", + "rustc-hash", "shlex", - "which 2.0.1", + "which", ] [[package]] @@ -423,13 +424,15 @@ dependencies = [ [[package]] name = "browser-utils" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ + "chrono", "clear_on_drop", "console_error_panic_hook", "console_log", "futures 0.1.29", "futures 0.3.4", + "futures-timer 3.0.1", "js-sys", "kvdb-web", "libp2p", @@ -437,12 +440,19 @@ dependencies = [ "rand 0.6.5", "rand 0.7.3", "sc-chain-spec", + "sc-informant", "sc-network", "sc-service", "wasm-bindgen", "wasm-bindgen-futures", ] +[[package]] +name = "bs58" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95ee6bba9d950218b6cc910cf62bc9e0a171d0f4537e3627b0f54d08549b188" + [[package]] name = "bs58" version = "0.3.0" @@ -472,9 +482,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.1.2" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb8038c1ddc0a5f73787b130f4cc75151e96ed33e417fde765eb5a81e3532f4" +checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" [[package]] name = "byte-slice-cast" @@ -577,9 +587,18 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chacha20-poly1305-aead" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d2058ba29594f69c75e8a9018e0485e3914ca5084e3613cd64529042f5423b" +dependencies = [ + "constant_time_eq", +] + [[package]] name = "chain-spec-builder" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "ansi_term 0.12.1", "node-cli", @@ -595,10 +614,11 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31850b4a4d6bae316f7a09e691c944c28299298837edc0a03f755618c23cbc01" dependencies = [ + "js-sys", "num-integer", "num-traits", - "serde", "time", + "wasm-bindgen", ] [[package]] @@ -718,24 +738,25 @@ checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" [[package]] name = "cranelift-bforest" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd05aac8cefcde54ce26178df8f36cb1f518ac691db650e7d2440c2b6b41c4dc" +checksum = "fd0f53d59dc9ab1c8ab68c991d8406b52b7a0aab0b15b05a3a6895579c4e5dd9" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63d9b6ff8a94f98deabab21880d7fd54996e0e16be687b6f80a3b6bdd9c188d" +checksum = "0381a794836fb994c47006465d46d46be072483b667f36013d993b9895117fee" dependencies = [ "byteorder 1.3.4", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", "cranelift-entity", + "gimli 0.20.0", "log 0.4.8", "serde", "smallvec 1.2.0", @@ -745,9 +766,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb3df51c2c07d719d02869bfac6cabd8d82ee308d5b29ca62e6528723cc33a4" +checksum = "208c3c8d82bfef32a534c5020c6cfc3bc92f41388f1246b7bb98cf543331abaa" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -755,24 +776,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758f9426b2e22bf83fc1a6b231a9d53cd4830751883c7f0e196ebb3c210467b3" +checksum = "ea048c456a517e56fd6df8f0e3947922897e6e6f61fbc5eb557a36c7b8ff6394" [[package]] name = "cranelift-entity" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff064733df8b98f453060264a8790393d1e807aca6942706b42f79a4f7aae9ed" +checksum = "0c8c7ed50812194c9e9de1fa39c77b39fc9ab48173d5e7ee88b25b6a8953e9b8" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eaafb5fa623dcbe19a28084a8226d7a1b17184a949c1a1f29a46b479867998d" +checksum = "21ceb931d9f919731df1b1ecdc716b5c66384b413a7f95909d1f45441ab9bef5" dependencies = [ "cranelift-codegen", "log 0.4.8", @@ -782,9 +803,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90033dbd7293f6fad4cf9dcd769cd621d60df22b1c5a11799e86359b7447a51d" +checksum = "564ee82268bc25b914fcf331edfc2452f2d9ca34f976b187b4ca668beba250c8" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -793,9 +814,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cb82a1071f88822763a583ec1a8688ffe5e2cda02c111d4483dd4376ed14d8" +checksum = "de63e2271b374be5b07f359184e2126a08fb24d24a740cbc178b7e0107ddafa5" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -803,7 +824,7 @@ dependencies = [ "log 0.4.8", "serde", "thiserror", - "wasmparser", + "wasmparser 0.48.2", ] [[package]] @@ -918,7 +939,7 @@ dependencies = [ "crossbeam-utils", "lazy_static", "memoffset", - "scopeguard 1.0.0", + "scopeguard", ] [[package]] @@ -995,7 +1016,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -1057,8 +1078,8 @@ version = "0.99.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2159be042979966de68315bce7034bb000c775f22e3e834e1c52ff78f041cae8" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1148,8 +1169,8 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1229,9 +1250,9 @@ dependencies = [ [[package]] name = "evm" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f887b371f9999682ccc5b1cb771e7d4408ae61e93fc0343ceaeb761fca42d1" +checksum = "272f65e18a2b6449b682bfcdf6c3ccc63db0b93898d89c0fb237548bbfc764a5" dependencies = [ "evm-core", "evm-gasometer", @@ -1244,18 +1265,18 @@ dependencies = [ [[package]] name = "evm-core" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcde5af3d542874ddeb53de0919302d57586ea04b3f76f54d865f8a6cdc70ae" +checksum = "66534d42e13d50f9101bed87cb568fd5aa929c600c3c13f8dadbbf39f5635945" dependencies = [ "primitive-types", ] [[package]] name = "evm-gasometer" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82bc9f275cb59d2bcc05d85c98736ddfaba003a7ef7b73893fa7c1c1fab29dc" +checksum = "39bc5b592803ca644781fe2290b7305ea5182f7c9516d615ddfb2308c2cab639" dependencies = [ "evm-core", "evm-runtime", @@ -1264,9 +1285,9 @@ dependencies = [ [[package]] name = "evm-runtime" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dbbc89d29618c3722c17ba78ddf432d40ace8ee27e3f8b28b52a85921112e4b" +checksum = "389e4b447fb26971a9c76c8aa49c0ab435f8e46e8fc46e1bc4ebf01f3c2b428f" dependencies = [ "evm-core", "primitive-types", @@ -1284,9 +1305,9 @@ dependencies = [ [[package]] name = "faerie" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f902f2af041f6c7177a2a04f805687cdc71e69c7cbef059a2755d8923f4cd7a8" +checksum = "74b9ed6159e4a6212c61d9c6a86bee01876b192a64accecf58d5b5ae3b667b52" dependencies = [ "anyhow", "goblin", @@ -1314,8 +1335,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "synstructure", ] @@ -1421,14 +1442,40 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "fork-tree" -version = "2.0.0" +version = "2.0.0-alpha.1" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "frame-benchmarking" +version = "2.0.0-alpha.1" +dependencies = [ + "parity-scale-codec", + "sp-api", + "sp-io", + "sp-runtime-interface", + "sp-std", +] + +[[package]] +name = "frame-benchmarking-cli" +version = "2.0.0-alpha.1" dependencies = [ + "frame-benchmarking", "parity-scale-codec", + "sc-cli", + "sc-client", + "sc-client-db", + "sc-executor", + "sc-service", + "sp-runtime", + "structopt", ] [[package]] name = "frame-executive" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -1446,7 +1493,7 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "11.0.0" +version = "11.0.0-alpha.1" dependencies = [ "parity-scale-codec", "serde", @@ -1456,7 +1503,7 @@ dependencies = [ [[package]] name = "frame-support" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "bitmask", "frame-metadata", @@ -1464,7 +1511,7 @@ dependencies = [ "frame-system", "impl-trait-for-tuples", "log 0.4.8", - "once_cell 0.2.4", + "once_cell", "parity-scale-codec", "paste", "pretty_assertions", @@ -1481,37 +1528,37 @@ dependencies = [ [[package]] name = "frame-support-procedural" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support-procedural-tools", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-procedural-tools" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-procedural-tools-derive" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "frame-support", "parity-scale-codec", @@ -1527,7 +1574,7 @@ dependencies = [ [[package]] name = "frame-system" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "criterion 0.2.11", "frame-support", @@ -1545,7 +1592,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-api", @@ -1573,6 +1620,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1698,8 +1751,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1717,20 +1770,19 @@ checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" [[package]] name = "futures-timer" -version = "0.4.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878f1d2fc31355fa02ed2372e741b0c17e58373341e6a122569b4623a14a7d33" -dependencies = [ - "futures-core-preview", - "futures-util-preview", - "pin-utils", -] +checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" [[package]] name = "futures-timer" -version = "2.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" +checksum = "3de1a2b2a2a33d9e60e17980b60ee061eeaae96a5abe9121db0fdb9af167a1c5" +dependencies = [ + "gloo-timers", + "send_wrapper 0.4.0", +] [[package]] name = "futures-util" @@ -1777,15 +1829,6 @@ dependencies = [ "pin-project", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder 1.3.4", -] - [[package]] name = "gcc" version = "0.3.55" @@ -1848,6 +1891,16 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "gimli" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633" +dependencies = [ + "byteorder 1.3.4", + "indexmap", +] + [[package]] name = "glob" version = "0.2.11" @@ -1873,6 +1926,19 @@ dependencies = [ "regex", ] +[[package]] +name = "gloo-timers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2d17dbd803c2fc86cb1b613adf63192046a7176f383a8302594654752c4c4a" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "goblin" version = "0.1.3" @@ -1884,34 +1950,6 @@ dependencies = [ "scroll", ] -[[package]] -name = "grafana-data-source" -version = "0.8.0" -dependencies = [ - "async-std", - "chrono", - "derive_more", - "futures-timer 2.0.2", - "futures-util", - "hyper 0.13.2", - "lazy_static", - "log 0.4.8", - "parking_lot 0.10.0", - "serde", - "serde_json", - "tokio 0.2.11", -] - -[[package]] -name = "grafana-data-source-test" -version = "2.0.0" -dependencies = [ - "futures 0.3.4", - "futures-timer 2.0.2", - "grafana-data-source", - "rand 0.7.3", -] - [[package]] name = "h2" version = "0.1.26" @@ -1964,22 +2002,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "hashbrown" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" -dependencies = [ - "byteorder 1.3.4", - "scopeguard 0.3.3", -] - -[[package]] -name = "hashbrown" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1de41fb8dba9714efd92241565cdff73f78508c95697dd56787d3cba27e2353" - [[package]] name = "hashbrown" version = "0.6.3" @@ -2010,9 +2032,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" +checksum = "76cdda6bf525062a0c9e8f14ee2b37935c86b8efb6c8b69b3c83dfb518a914af" [[package]] name = "hex-literal" @@ -2178,6 +2200,7 @@ dependencies = [ "httparse", "itoa", "log 0.4.8", + "net2", "pin-project", "time", "tokio 0.2.11", @@ -2187,19 +2210,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.17.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719d85c7df4a7f309a77d145340a063ea929dcb2e025bae46a80345cffec2952" +checksum = "f6ea6215c7314d450ee45970ab8b3851ab447a0e6bafdd19e31b20a42dbb7faf" dependencies = [ - "bytes 0.4.12", + "bytes 0.5.4", "ct-logs", - "futures 0.1.29", - "hyper 0.12.35", + "futures-util", + "hyper 0.13.2", "rustls", - "tokio-io", + "rustls-native-certs", + "tokio 0.2.11", "tokio-rustls", "webpki", - "webpki-roots 0.17.0", ] [[package]] @@ -2279,8 +2302,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -2402,35 +2425,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8609af8f63b626e8e211f52441fcdb6ec54f1a446606b10d5c89ae9bf8a20058" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "jsonrpc-http-server" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d83d348120edee487c560b7cdd2565055d61cda053aa0d0ef0f8b6a18429048" +checksum = "816d63997ea45d3634608edbef83ddb35e661f7c0b27b5b72f237e321f0e9807" dependencies = [ "hyper 0.12.35", "jsonrpc-core", "jsonrpc-server-utils", "log 0.4.8", "net2", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "unicase 2.6.0", ] [[package]] name = "jsonrpc-pubsub" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3453625f0f0f5cd6d6776d389d73b7d70fcc98620b7cbb1cbbb1f6a36e95f39a" +checksum = "5b31c9b90731276fdd24d896f31bb10aecf2e5151733364ae81123186643d939" dependencies = [ "jsonrpc-core", "log 0.4.8", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "serde", ] @@ -2452,14 +2475,14 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34faa167c3ac9705aeecb986c0da6056529f348425dbe0441db60a2c4cc41d1" +checksum = "b94e5773b2ae66e0e02c80775ce6bbba6f15d5bb47c14ec36a36fcf94f8df851" dependencies = [ "jsonrpc-core", "jsonrpc-server-utils", "log 0.4.8", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "slab", "ws", ] @@ -2576,11 +2599,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + [[package]] name = "libc" -version = "0.2.66" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" [[package]] name = "libloading" @@ -2594,9 +2623,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.15.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84847789ab24b3fc5971a68656ac85886df640986d9ce3264c0327694eae471" +checksum = "f6bf152b510950e1030f2d3dcca5f0b4017892be50348a15fd3eec8b90c826fb" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2614,6 +2643,7 @@ dependencies = [ "libp2p-noise", "libp2p-ping", "libp2p-plaintext", + "libp2p-pnet", "libp2p-secio", "libp2p-swarm", "libp2p-tcp", @@ -2621,8 +2651,8 @@ dependencies = [ "libp2p-wasm-ext", "libp2p-websocket", "libp2p-yamux", - "parity-multiaddr", - "parity-multihash", + "parity-multiaddr 0.7.2", + "parity-multihash 0.2.3", "parking_lot 0.10.0", "pin-project", "smallvec 1.2.0", @@ -2631,22 +2661,22 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbafb2706b8082233f66dc13e196f9cf9b4c229f2cd7c96b2b16617ad6ee330b" +checksum = "3b874594c4b29de1a29f27871feba8e6cd13aa54a8a1e8f8c7cf3dfac5ca287c" dependencies = [ "asn1_der", - "bs58", + "bs58 0.3.0", "ed25519-dalek", "fnv", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "lazy_static", "libsecp256k1", "log 0.4.8", "multistream-select", - "parity-multiaddr", - "parity-multihash", + "parity-multiaddr 0.7.2", + "parity-multihash 0.2.3", "parking_lot 0.10.0", "pin-project", "prost", @@ -2657,27 +2687,26 @@ dependencies = [ "sha2", "smallvec 1.2.0", "thiserror", - "unsigned-varint", - "untrusted", + "unsigned-varint 0.3.1", "void", "zeroize 1.1.0", ] [[package]] name = "libp2p-core-derive" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c320266be0a7760e23484d635acdb83844b2d74d3612d93b41c393c9bcf004e" +checksum = "96d472e9d522f588805c77801de10b957be84e10f019ca5f869fa1825b15ea9b" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] [[package]] name = "libp2p-deflate" -version = "0.7.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be32697b42d040b325c3737f827ea04ede569ec956b7807700dd8d89d8210f9" +checksum = "2e25004d4d9837b44b22c5f1a69be1724a5168fef6cff1716b5176a972c3aa62" dependencies = [ "flate2", "futures 0.3.4", @@ -2686,9 +2715,9 @@ dependencies = [ [[package]] name = "libp2p-dns" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c979b882f25d85726b15637d5bbc722dfa1be576605c54e99b8cf56906be3" +checksum = "b99e552f9939b606eb4b59f7f64d9b01e3f96752f47e350fc3c5fc646ed3f649" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2697,11 +2726,10 @@ dependencies = [ [[package]] name = "libp2p-floodsub" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bdf6fba9272ad47dde94bade89540fdb16e24ae9ff7fb714c1c80a035165f28" +checksum = "1d3234f12e44f9a50351a9807b97fe7de11eb9ae4482370392ba10da6dc90722" dependencies = [ - "bs58", "cuckoofilter", "fnv", "futures 0.3.4", @@ -2715,12 +2743,11 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6ecd058bf769d27ebec530544b081e08b0a1088e3186da8cc58d59915784d0" +checksum = "d46cb3e0841bd951cbf4feae56cdc081e6347836a644fb260c3ec554149b4006" dependencies = [ "base64 0.11.0", - "bs58", "byteorder 1.3.4", "bytes 0.5.4", "fnv", @@ -2729,21 +2756,21 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.8", - "lru 0.4.3", + "lru", "prost", "prost-build", "rand 0.7.3", "sha2", "smallvec 1.2.0", - "unsigned-varint", + "unsigned-varint 0.3.1", "wasm-timer", ] [[package]] name = "libp2p-identify" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1a6261b804111c2dbf53f8ca03f66edc5ad1c29b78a61cf0cc5354052e28e9" +checksum = "bfeb935a9bd41263e4f3a24b988e9f4a044f3ae89ac284e83c17fe2f84e0d66b" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2757,9 +2784,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.15.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6ea6fece0d99599afb1b2082ca8937944cdd6b0946a88d54cb3ae7a38d1253" +checksum = "76c260a92309112fff855ab2bd8f26c246c1dd380b87021abe61358dedb9748d" dependencies = [ "arrayvec 0.5.1", "bytes 0.5.4", @@ -2770,23 +2797,23 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.8", - "parity-multihash", + "parity-multihash 0.2.3", "prost", "prost-build", "rand 0.7.3", "sha2", "smallvec 1.2.0", "uint", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", "wasm-timer", ] [[package]] name = "libp2p-mdns" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074312353355df310affa105ec71b16fd7e52f5c6ae61d3dcbb3e79e8fdc9e5f" +checksum = "881fcfb360c2822db9f0e6bb6f89529621556ed9a8b038313414eda5107334de" dependencies = [ "async-std", "data-encoding", @@ -2806,9 +2833,9 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d0b44dfdef80cc2be4b42d127de1c793905eca2459415a5c57d6b4fbd8ec30" +checksum = "d8507b37ad0eed275efcde67a023c3d85af6c80768b193845b9288e848e1af95" dependencies = [ "bytes 0.5.4", "fnv", @@ -2817,14 +2844,14 @@ dependencies = [ "libp2p-core", "log 0.4.8", "parking_lot 0.10.0", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] name = "libp2p-noise" -version = "0.13.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845e8208d814cd41c26c90a6a2f2b720c31b588209cecc49a44c881a09f417f" +checksum = "ac7d33809afdf6794f09fdb2f9f94e1550ae230be5bae6430a078eb96fc9e5a6" dependencies = [ "curve25519-dalek 1.2.3", "futures 0.3.4", @@ -2834,17 +2861,18 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "ring", + "sha2", "snow", - "x25519-dalek", + "static_assertions", + "x25519-dalek 0.5.2", "zeroize 1.1.0", ] [[package]] name = "libp2p-ping" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ecced2949ae93b6ff29565303ecd1bef15c4e4efb689033ee744922561a36b" +checksum = "33d22f2f228b3a828dca1cb8aa9fa331e0bc9c36510cb2c1916956e20dc85e8c" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2857,9 +2885,9 @@ dependencies = [ [[package]] name = "libp2p-plaintext" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195fda6b6a948a242fd30570e0e3418ae8e0a20055ea75d45458e1079a8efb05" +checksum = "56126a204d7b3382bac163143ff4125a14570b3ba76ba979103d1ae1abed1923" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2869,15 +2897,29 @@ dependencies = [ "prost", "prost-build", "rw-stream-sink", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", ] +[[package]] +name = "libp2p-pnet" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b916938a8868f75180aeeffcc6a516a922d165e8fa2a90b57bad989d1ccbb57a" +dependencies = [ + "futures 0.3.4", + "log 0.4.8", + "pin-project", + "rand 0.7.3", + "salsa20", + "sha3", +] + [[package]] name = "libp2p-secio" -version = "0.15.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceef68ca377b264f84d64c88739a8fa118b5db1e8f18297351dff75314504a5f" +checksum = "1219e9ecb4945d7331a05f5ffe96a1f6e28051bfa1223d4c60353c251de0354e" dependencies = [ "aes-ctr", "ctr", @@ -2905,9 +2947,9 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.5.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ea00be81bc3985e36abad263ce2ad1b6aeb862aa743563eb70ad42880c05ae" +checksum = "275471e7c0e88ae004660866cd54f603bd8bd1f4caef541a27f50dd8640c4d4c" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2919,13 +2961,13 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e65ef381570df31cb047dfbc11483ab0fe7e6abbdcf2bdc2c60b5d11133d241" +checksum = "f9e80ad4e3535345f3d666554ce347d3100453775611c05c60786bf9a1747a10" dependencies = [ "async-std", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "get_if_addrs", "ipnet", "libp2p-core", @@ -2934,9 +2976,9 @@ dependencies = [ [[package]] name = "libp2p-uds" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4f4f7989b35f33d4b9738aab2f031310eb20fec513cab44d12b1bc985a8074" +checksum = "76d329564a43da9d0e055a5b938633c4a8ceab1f59cec133fbc4647917c07341" dependencies = [ "async-std", "futures 0.3.4", @@ -2946,9 +2988,9 @@ dependencies = [ [[package]] name = "libp2p-wasm-ext" -version = "0.8.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b4d457adb91a5e2212343218a554394cd8ced64a79fb8e36e7aed2a16d49495" +checksum = "7d40c95ac1a9b7fb7770616e4165f34559885337f3be485b32acdd085261be3a" dependencies = [ "futures 0.3.4", "js-sys", @@ -2960,9 +3002,9 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba425f2af1fdb7dece88b9ae05ca9430dfb0b72b2c078e73ded6f1556084509" +checksum = "5351ca9eea122081c1c0f9323164d2918cac29b5a6bfe5054d4ba8ec9447cf42" dependencies = [ "async-tls", "bytes 0.5.4", @@ -2981,13 +3023,12 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca25b3aac78a3c93c2a567622abd3cfc16f96f26ae1bf6134f0056203d62d86" +checksum = "f72aa5a7273c29c6eaea09108a49feaefc7456164863f64f86a193f9e78a4b7f" dependencies = [ "futures 0.3.4", "libp2p-core", - "log 0.4.8", "parking_lot 0.10.0", "thiserror", "yamux", @@ -2995,9 +3036,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.2.4" +version = "6.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be" +checksum = "4e3b727e2dd20ec2fb7ed93f23d9fd5328a0871185485ebdaff007b47d3e27e4" dependencies = [ "bindgen", "cc", @@ -3048,22 +3089,13 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "lock_api" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" -dependencies = [ - "scopeguard 0.3.3", -] - [[package]] name = "lock_api" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" dependencies = [ - "scopeguard 1.0.0", + "scopeguard", ] [[package]] @@ -3084,22 +3116,13 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "lru" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8f669d42c72d18514dfca8115689c5f6370a17d980cb5bd777a67f404594c8" -dependencies = [ - "hashbrown 0.5.0", -] - [[package]] name = "lru" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0609345ddee5badacf857d4f547e0e5a2e987db77085c24cd887f73573a04237" dependencies = [ - "hashbrown 0.6.3", + "hashbrown", ] [[package]] @@ -3125,9 +3148,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.3.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" dependencies = [ "libc", ] @@ -3149,7 +3172,7 @@ checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" dependencies = [ "ahash", "hash-db", - "hashbrown 0.6.3", + "hashbrown", "parity-util-mem", ] @@ -3266,7 +3289,7 @@ dependencies = [ "log 0.4.8", "smallvec 1.2.0", "tokio-io", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] @@ -3322,10 +3345,11 @@ dependencies = [ [[package]] name = "node-cli" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "assert_cmd", "browser-utils", + "frame-benchmarking-cli", "frame-support", "frame-system", "futures 0.3.4", @@ -3334,6 +3358,7 @@ dependencies = [ "log 0.4.8", "nix", "node-executor", + "node-inspect", "node-primitives", "node-rpc", "node-runtime", @@ -3367,6 +3392,7 @@ dependencies = [ "sc-tracing", "sc-transaction-pool", "serde", + "serde_json", "sp-authority-discovery", "sp-consensus", "sp-consensus-babe", @@ -3390,9 +3416,10 @@ dependencies = [ [[package]] name = "node-executor" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "criterion 0.3.1", + "frame-benchmarking", "frame-support", "frame-system", "node-primitives", @@ -3416,13 +3443,29 @@ dependencies = [ "sp-state-machine", "sp-trie", "substrate-test-client", - "trie-root 0.15.2", + "trie-root", "wabt", ] +[[package]] +name = "node-inspect" +version = "0.8.0-alpha.1" +dependencies = [ + "derive_more", + "log 0.4.8", + "parity-scale-codec", + "sc-cli", + "sc-client-api", + "sc-service", + "sp-blockchain", + "sp-core", + "sp-runtime", + "structopt", +] + [[package]] name = "node-primitives" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "pretty_assertions", "sp-core", @@ -3432,7 +3475,7 @@ dependencies = [ [[package]] name = "node-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "jsonrpc-core", "node-primitives", @@ -3440,7 +3483,14 @@ dependencies = [ "pallet-contracts-rpc", "pallet-transaction-payment-rpc", "sc-client", + "sc-consensus-babe", + "sc-consensus-babe-rpc", + "sc-consensus-epochs", + "sc-keystore", "sp-api", + "sp-blockchain", + "sp-consensus", + "sp-consensus-babe", "sp-runtime", "sp-transaction-pool", "substrate-frame-rpc-system", @@ -3448,7 +3498,7 @@ dependencies = [ [[package]] name = "node-rpc-client" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "env_logger 0.7.1", "futures 0.1.29", @@ -3461,8 +3511,9 @@ dependencies = [ [[package]] name = "node-runtime" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ + "frame-benchmarking", "frame-executive", "frame-support", "frame-system", @@ -3522,7 +3573,7 @@ dependencies = [ [[package]] name = "node-template" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "futures 0.3.4", "log 0.4.8", @@ -3550,7 +3601,7 @@ dependencies = [ [[package]] name = "node-template-runtime" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-executive", "frame-support", @@ -3558,7 +3609,6 @@ dependencies = [ "pallet-aura", "pallet-balances", "pallet-grandpa", - "pallet-indices", "pallet-randomness-collective-flip", "pallet-sudo", "pallet-template", @@ -3583,10 +3633,13 @@ dependencies = [ [[package]] name = "node-testing" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ + "criterion 0.3.1", "frame-support", "frame-system", + "fs_extra", + "log 0.4.8", "node-executor", "node-primitives", "node-runtime", @@ -3601,19 +3654,31 @@ dependencies = [ "pallet-transaction-payment", "pallet-treasury", "parity-scale-codec", + "sc-cli", "sc-client", + "sc-client-api", + "sc-client-db", "sc-executor", + "sc-service", + "sp-api", + "sp-block-builder", + "sp-blockchain", + "sp-consensus", "sp-core", + "sp-finality-tracker", + "sp-inherents", "sp-io", "sp-keyring", "sp-runtime", + "sp-timestamp", "substrate-test-client", + "tempfile", "wabt", ] [[package]] name = "node-transaction-factory" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "log 0.4.8", "parity-scale-codec", @@ -3637,9 +3702,9 @@ checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] name = "nohash-hasher" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721a2bf1c26159ebf17e0a980bc4ce61f4b2fec5ec3b42d42fddd7a84a9e538f" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" @@ -3713,26 +3778,14 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "once_cell" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532c29a261168a45ce28948f9537ddd7a5dd272cc513b3017b1e82a88f962c37" -dependencies = [ - "parking_lot 0.7.1", -] - -[[package]] -name = "once_cell" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d584f08c2d717d5c23a6414fc2822b71c651560713e54fa7eace675f758a355e" - [[package]] name = "once_cell" version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +dependencies = [ + "parking_lot 0.9.0", +] [[package]] name = "oorandom" @@ -3799,7 +3852,7 @@ dependencies = [ [[package]] name = "pallet-assets" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3813,7 +3866,7 @@ dependencies = [ [[package]] name = "pallet-aura" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3835,7 +3888,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3853,7 +3906,7 @@ dependencies = [ [[package]] name = "pallet-authorship" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3869,7 +3922,7 @@ dependencies = [ [[package]] name = "pallet-babe" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3894,8 +3947,9 @@ dependencies = [ [[package]] name = "pallet-balances" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "pallet-transaction-payment", @@ -3909,7 +3963,7 @@ dependencies = [ [[package]] name = "pallet-collective" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -3925,7 +3979,7 @@ dependencies = [ [[package]] name = "pallet-contracts" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "assert_matches", "frame-support", @@ -3950,7 +4004,7 @@ dependencies = [ [[package]] name = "pallet-contracts-primitives" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -3959,7 +4013,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -3978,7 +4032,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "pallet-contracts-primitives", "parity-scale-codec", @@ -3989,7 +4043,7 @@ dependencies = [ [[package]] name = "pallet-democracy" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4006,7 +4060,7 @@ dependencies = [ [[package]] name = "pallet-elections" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4022,7 +4076,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4040,7 +4094,7 @@ dependencies = [ [[package]] name = "pallet-evm" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "evm", "frame-support", @@ -4060,7 +4114,7 @@ dependencies = [ [[package]] name = "pallet-example" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4073,9 +4127,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-example-offchain-worker" +version = "2.0.0-alpha.1" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "serde", + "serde_json", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-finality-tracker" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4092,7 +4161,7 @@ dependencies = [ [[package]] name = "pallet-generic-asset" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4106,7 +4175,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4124,9 +4193,10 @@ dependencies = [ [[package]] name = "pallet-identity" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "enumflags2", + "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", @@ -4140,7 +4210,7 @@ dependencies = [ [[package]] name = "pallet-im-online" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4158,10 +4228,11 @@ dependencies = [ [[package]] name = "pallet-indices" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", + "pallet-balances", "parity-scale-codec", "serde", "sp-core", @@ -4173,7 +4244,7 @@ dependencies = [ [[package]] name = "pallet-membership" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4187,7 +4258,7 @@ dependencies = [ [[package]] name = "pallet-nicks" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4202,7 +4273,7 @@ dependencies = [ [[package]] name = "pallet-offences" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4218,7 +4289,7 @@ dependencies = [ [[package]] name = "pallet-randomness-collective-flip" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4232,7 +4303,7 @@ dependencies = [ [[package]] name = "pallet-recovery" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "enumflags2", "frame-support", @@ -4248,7 +4319,7 @@ dependencies = [ [[package]] name = "pallet-scored-pool" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4263,7 +4334,7 @@ dependencies = [ [[package]] name = "pallet-session" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4283,7 +4354,7 @@ dependencies = [ [[package]] name = "pallet-society" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4299,7 +4370,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4322,18 +4393,18 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "sp-runtime", "syn", ] [[package]] name = "pallet-sudo" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4347,7 +4418,7 @@ dependencies = [ [[package]] name = "pallet-template" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4360,8 +4431,9 @@ dependencies = [ [[package]] name = "pallet-timestamp" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "impl-trait-for-tuples", @@ -4377,7 +4449,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4392,7 +4464,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4409,7 +4481,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "parity-scale-codec", @@ -4422,7 +4494,7 @@ dependencies = [ [[package]] name = "pallet-treasury" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4437,7 +4509,7 @@ dependencies = [ [[package]] name = "pallet-utility" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -4452,7 +4524,7 @@ dependencies = [ [[package]] name = "pallet-vesting" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "enumflags2", "frame-support", @@ -4476,27 +4548,60 @@ checksum = "0c276d76c5333b8c2579e02d49a06733a55b8282d2d9b13e8d53b6406bd7e30a" [[package]] name = "parity-multiaddr" -version = "0.7.1" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045b3c7af871285146300da35b1932bb6e4639b66c7c98e85d06a32cbc4e8fa7" +dependencies = [ + "arrayref", + "bs58 0.2.5", + "byteorder 1.3.4", + "bytes 0.4.12", + "data-encoding", + "parity-multihash 0.1.3", + "percent-encoding 1.0.1", + "serde", + "unsigned-varint 0.2.3", + "url 1.7.2", +] + +[[package]] +name = "parity-multiaddr" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80878c27f90dd162d3143333d672e80b194d6b080f05c83440e3dfda42e409f2" +checksum = "26df883298bc3f4e92528b4c5cc9f806b791955b136da3e5e939ed9de0fd958b" dependencies = [ "arrayref", - "bs58", + "bs58 0.3.0", "byteorder 1.3.4", "data-encoding", - "parity-multihash", + "parity-multihash 0.2.3", "percent-encoding 2.1.0", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.3.1", "url 2.1.1", ] [[package]] name = "parity-multihash" -version = "0.2.2" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3a17dc27848fd99e4f87eb0f8c9baba6ede0a6d555400c850ca45254ef4ce3" +dependencies = [ + "blake2", + "bytes 0.4.12", + "rand 0.6.5", + "sha-1", + "sha2", + "sha3", + "unsigned-varint 0.2.3", +] + +[[package]] +name = "parity-multihash" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11f42bbd3a021c5061b77154bd3334d5a57e1a03eb162de0b962681cc25800d" +checksum = "7a1cd2ba02391b81367bec529fb209019d718684fdc8ad6a712c2b536e46f775" dependencies = [ "blake2", "bytes 0.5.4", @@ -4504,7 +4609,7 @@ dependencies = [ "sha-1", "sha2", "sha3", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] @@ -4527,8 +4632,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34e513ff3e406f3ede6796dcdc83d0b32ffb86668cea1ccf7363118abeb00476" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4559,7 +4664,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2", "syn", "synstructure", ] @@ -4579,23 +4684,13 @@ version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" -[[package]] -name = "parking_lot" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" -dependencies = [ - "lock_api 0.1.5", - "parking_lot_core 0.4.0", -] - [[package]] name = "parking_lot" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" dependencies = [ - "lock_api 0.3.3", + "lock_api", "parking_lot_core 0.6.2", "rustc_version", ] @@ -4606,23 +4701,10 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" dependencies = [ - "lock_api 0.3.3", + "lock_api", "parking_lot_core 0.7.0", ] -[[package]] -name = "parking_lot_core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" -dependencies = [ - "libc", - "rand 0.6.5", - "rustc_version", - "smallvec 0.6.13", - "winapi 0.3.8", -] - [[package]] name = "parking_lot_core" version = "0.6.2" @@ -4669,8 +4751,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4214c9e912ef61bf42b81ba9a47e8aad1b2ffaf739ab162bf96d1e011f54e6c5" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4733,8 +4815,8 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4847,8 +4929,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "875077759af22fa20b610ad4471d8155b321c89c3f2785526c9839b099be4e0a" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "rustversion", "syn", ] @@ -4859,8 +4941,8 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5717d9fa2664351a01ed73ba5ef6df09c01a521cb42cb65a061432a826f3c7a" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "rustversion", "syn", "syn-mid", @@ -4872,8 +4954,8 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4885,20 +4967,38 @@ checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" [[package]] name = "proc-macro2" -version = "0.4.30" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" dependencies = [ - "unicode-xid 0.1.0", + "unicode-xid", ] [[package]] -name = "proc-macro2" -version = "1.0.8" +name = "prometheus" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +checksum = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "protobuf", + "quick-error", + "spin", +] + +[[package]] +name = "prometheus-exporter" +version = "0.8.0-alpha.1" dependencies = [ - "unicode-xid 0.2.0", + "async-std", + "derive_more", + "futures-util", + "hyper 0.13.2", + "log 0.4.8", + "prometheus", + "tokio 0.2.11", ] [[package]] @@ -4926,7 +5026,7 @@ dependencies = [ "prost", "prost-types", "tempfile", - "which 3.1.0", + "which", ] [[package]] @@ -4937,8 +5037,8 @@ checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" dependencies = [ "anyhow", "itertools", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4952,6 +5052,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6686ddd96a8dbe2687b5f2a687b2cfb520854010ec480f2d74c32e7c9873d3c5" + [[package]] name = "pwasm-utils" version = "0.12.0" @@ -4992,22 +5098,13 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - [[package]] name = "quote" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2", ] [[package]] @@ -5362,6 +5459,12 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hex" version = "2.1.0" @@ -5390,14 +5493,26 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls-native-certs" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ffebdbb48c14f84eba0b715197d673aff1dd22cc1007ca647e28483bbcc307" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "rustversion" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -5433,6 +5548,26 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "salsa20" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2324b0e8c3bb9a586a571fdb3136f70e7e2c748de00a78043f86e0cff91f91fe" +dependencies = [ + "byteorder 1.3.4", + "salsa20-core", + "stream-cipher", +] + +[[package]] +name = "salsa20-core" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe6cc1b9f5a5867853ade63099de70f042f7679e408d1ffe52821c9248e6e69" +dependencies = [ + "stream-cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -5444,13 +5579,13 @@ dependencies = [ [[package]] name = "sc-authority-discovery" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ - "bytes 0.4.12", + "bytes 0.5.4", "derive_more", "env_logger 0.7.1", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -5473,7 +5608,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "futures 0.3.4", "log 0.4.8", @@ -5497,7 +5632,7 @@ dependencies = [ [[package]] name = "sc-block-builder" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -5512,7 +5647,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-trait-for-tuples", "sc-chain-spec-derive", @@ -5526,17 +5661,17 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sc-cli" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "ansi_term 0.12.1", "app_dirs", @@ -5551,9 +5686,11 @@ dependencies = [ "log 0.4.8", "names", "parity-util-mem", + "prometheus-exporter", "regex", "rpassword", "sc-client-api", + "sc-informant", "sc-network", "sc-service", "sc-telemetry", @@ -5573,7 +5710,7 @@ dependencies = [ [[package]] name = "sc-client" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5610,7 +5747,7 @@ dependencies = [ [[package]] name = "sc-client-api" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "fnv", @@ -5641,7 +5778,7 @@ dependencies = [ [[package]] name = "sc-client-db" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "env_logger 0.7.1", "hash-db", @@ -5654,6 +5791,7 @@ dependencies = [ "parity-util-mem", "parking_lot 0.10.0", "quickcheck", + "rand 0.7.3", "sc-client", "sc-client-api", "sc-executor", @@ -5671,13 +5809,13 @@ dependencies = [ [[package]] name = "sc-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "env_logger 0.7.1", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5710,14 +5848,14 @@ dependencies = [ [[package]] name = "sc-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "env_logger 0.7.1", "fork-tree", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "log 0.4.8", "merlin", "num-bigint", @@ -5740,6 +5878,7 @@ dependencies = [ "sc-service", "sc-telemetry", "schnorrkel", + "serde", "sp-api", "sp-application-crypto", "sp-block-builder", @@ -5758,9 +5897,34 @@ dependencies = [ "tokio 0.1.22", ] +[[package]] +name = "sc-consensus-babe-rpc" +version = "0.8.0-alpha.1" +dependencies = [ + "derive_more", + "futures 0.3.4", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "sc-consensus-babe", + "sc-consensus-epochs", + "sc-keystore", + "serde", + "sp-api", + "sp-application-crypto", + "sp-blockchain", + "sp-consensus", + "sp-consensus-babe", + "sp-core", + "sp-keyring", + "sp-runtime", + "substrate-test-runtime-client", + "tempfile", +] + [[package]] name = "sc-consensus-epochs" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "fork-tree", "parity-scale-codec", @@ -5772,7 +5936,7 @@ dependencies = [ [[package]] name = "sc-consensus-manual-seal" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5800,7 +5964,7 @@ dependencies = [ [[package]] name = "sc-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "futures 0.3.4", @@ -5820,10 +5984,10 @@ dependencies = [ [[package]] name = "sc-consensus-slots" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5841,7 +6005,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "log 0.4.8", "sc-client-api", @@ -5854,7 +6018,7 @@ dependencies = [ [[package]] name = "sc-executor" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "assert_matches", "derive_more", @@ -5887,7 +6051,7 @@ dependencies = [ [[package]] name = "sc-executor-common" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "log 0.4.8", @@ -5902,7 +6066,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "log 0.4.8", "parity-scale-codec", @@ -5917,14 +6081,9 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "assert_matches", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", "log 0.4.8", "parity-scale-codec", "parity-wasm 0.41.0", @@ -5934,14 +6093,12 @@ dependencies = [ "sp-runtime-interface", "sp-wasm-interface", "wasmi", - "wasmtime-environ", - "wasmtime-jit", - "wasmtime-runtime", + "wasmtime", ] [[package]] name = "sc-finality-grandpa" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "assert_matches", "env_logger 0.7.1", @@ -5949,7 +6106,7 @@ dependencies = [ "fork-tree", "futures 0.1.29", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5966,23 +6123,39 @@ dependencies = [ "sp-api", "sp-arithmetic", "sp-blockchain", - "sp-consensus", - "sp-consensus-babe", - "sp-core", - "sp-finality-grandpa", - "sp-finality-tracker", - "sp-inherents", - "sp-keyring", + "sp-consensus", + "sp-consensus-babe", + "sp-core", + "sp-finality-grandpa", + "sp-finality-tracker", + "sp-inherents", + "sp-keyring", + "sp-runtime", + "sp-state-machine", + "substrate-test-runtime-client", + "tempfile", + "tokio 0.1.22", +] + +[[package]] +name = "sc-informant" +version = "0.8.0-alpha.1" +dependencies = [ + "ansi_term 0.12.1", + "futures 0.3.4", + "log 0.4.8", + "parity-util-mem", + "sc-client-api", + "sc-network", + "sc-service", + "sp-blockchain", "sp-runtime", - "sp-state-machine", - "substrate-test-runtime-client", - "tempfile", - "tokio 0.1.22", + "wasm-timer", ] [[package]] name = "sc-keystore" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "hex", @@ -5997,8 +6170,10 @@ dependencies = [ [[package]] name = "sc-network" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ + "assert_matches", + "async-std", "bitflags", "bytes 0.5.4", "derive_more", @@ -6008,15 +6183,19 @@ dependencies = [ "fnv", "fork-tree", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "futures_codec", "libp2p", "linked-hash-map", "linked_hash_set", "log 0.4.8", - "lru 0.4.3", + "lru", + "nohash-hasher", "parity-scale-codec", "parking_lot 0.10.0", + "pin-project", + "prost", + "prost-build", "quickcheck", "rand 0.7.3", "rustc-hex", @@ -6037,37 +6216,39 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-test-primitives", - "substrate-test-client", + "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", - "unsigned-varint", + "thiserror", + "unsigned-varint 0.3.1", "void", + "wasm-timer", "zeroize 1.1.0", ] [[package]] name = "sc-network-gossip" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ - "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", - "lru 0.1.17", + "lru", "parking_lot 0.10.0", "sc-network", "sp-runtime", + "wasm-timer", ] [[package]] name = "sc-network-test" -version = "0.8.0" +version = "0.8.0-dev" dependencies = [ "env_logger 0.7.1", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6089,15 +6270,14 @@ dependencies = [ [[package]] name = "sc-offchain" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ - "bytes 0.4.12", + "bytes 0.5.4", "env_logger 0.7.1", "fnv", - "futures 0.1.29", "futures 0.3.4", - "futures-timer 2.0.2", - "hyper 0.12.35", + "futures-timer 3.0.1", + "hyper 0.13.2", "hyper-rustls", "log 0.4.8", "num_cpus", @@ -6116,23 +6296,24 @@ dependencies = [ "sp-transaction-pool", "substrate-test-runtime-client", "threadpool", - "tokio 0.1.22", + "tokio 0.2.11", ] [[package]] name = "sc-peerset" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "futures 0.3.4", "libp2p", "log 0.4.8", "rand 0.7.3", "serde_json", + "wasm-timer", ] [[package]] name = "sc-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "assert_matches", "futures 0.1.29", @@ -6169,7 +6350,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "futures 0.3.4", @@ -6184,13 +6365,14 @@ dependencies = [ "serde_json", "sp-core", "sp-rpc", + "sp-runtime", "sp-transaction-pool", "sp-version", ] [[package]] name = "sc-rpc-server" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "jsonrpc-core", "jsonrpc-http-server", @@ -6204,7 +6386,7 @@ dependencies = [ [[package]] name = "sc-runtime-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-allocator", "sp-core", @@ -6217,21 +6399,21 @@ dependencies = [ [[package]] name = "sc-service" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "exit-future", "futures 0.1.29", "futures 0.3.4", "futures-diagnose", - "futures-timer 2.0.2", - "grafana-data-source", + "futures-timer 3.0.1", "lazy_static", "log 0.4.8", - "parity-multiaddr", + "parity-multiaddr 0.5.0", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", + "prometheus-exporter", "sc-chain-spec", "sc-client", "sc-client-api", @@ -6264,13 +6446,13 @@ dependencies = [ "sysinfo", "target_info", "tokio 0.2.11", - "tokio-executor 0.1.10", "tracing", + "wasm-timer", ] [[package]] name = "sc-service-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "env_logger 0.7.1", "fdlimit", @@ -6290,7 +6472,7 @@ dependencies = [ [[package]] name = "sc-state-db" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "env_logger 0.7.1", "log 0.4.8", @@ -6301,11 +6483,11 @@ dependencies = [ [[package]] name = "sc-telemetry" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "bytes 0.5.4", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6317,14 +6499,14 @@ dependencies = [ "slog-scope", "take_mut", "void", + "wasm-timer", ] [[package]] name = "sc-tracing" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "erased-serde", - "grafana-data-source", "log 0.4.8", "parking_lot 0.10.0", "sc-telemetry", @@ -6337,30 +6519,34 @@ dependencies = [ [[package]] name = "sc-transaction-graph" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "assert_matches", "criterion 0.3.1", "derive_more", "futures 0.3.4", + "linked-hash-map", "log 0.4.8", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", "serde", + "sp-blockchain", "sp-core", "sp-runtime", "sp-transaction-pool", "substrate-test-runtime", + "wasm-timer", ] [[package]] name = "sc-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "futures 0.3.4", "futures-diagnose", + "futures-timer 2.0.2", "log 0.4.8", "parity-scale-codec", "parity-util-mem", @@ -6375,6 +6561,7 @@ dependencies = [ "sp-transaction-pool", "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", + "wasm-timer", ] [[package]] @@ -6404,12 +6591,6 @@ dependencies = [ "zeroize 0.9.3", ] -[[package]] -name = "scopeguard" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" - [[package]] name = "scopeguard" version = "1.0.0" @@ -6431,8 +6612,8 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -6504,6 +6685,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "686ef91cf020ad8d4aca9a7047641fd6add626b7b89e14546c2b6a76781cf822" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "serde" version = "1.0.104" @@ -6519,16 +6706,16 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "serde_json" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b01d7f0288608a01dca632cf1df859df6fd6ffa885300fc275ce2ba6221953" +checksum = "15913895b61e0be854afd32fd4163fcd2a3df34142cf2cb961b310ce694cbf90" dependencies = [ "itoa", "ryu", @@ -6649,8 +6836,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -6676,10 +6863,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb767eee7d257ba202f0b9b08673bc13b22281632ef45267b19f13100accd2f" dependencies = [ "arrayref", + "blake2-rfc", + "chacha20-poly1305-aead", + "rand 0.7.3", "rand_core 0.5.1", "ring", "rustc_version", + "sha2", "subtle 2.2.2", + "x25519-dalek 0.6.0", ] [[package]] @@ -6710,7 +6902,7 @@ checksum = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" [[package]] name = "sp-allocator" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "log 0.4.8", @@ -6721,7 +6913,7 @@ dependencies = [ [[package]] name = "sp-api" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "hash-db", "parity-scale-codec", @@ -6736,18 +6928,18 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "blake2-rfc", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-api-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "criterion 0.3.1", "parity-scale-codec", @@ -6764,7 +6956,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "serde", @@ -6775,7 +6967,7 @@ dependencies = [ [[package]] name = "sp-application-crypto-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-api", "sp-application-crypto", @@ -6786,7 +6978,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "criterion 0.3.1", "integer-sqrt", @@ -6801,7 +6993,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-api", @@ -6812,7 +7004,7 @@ dependencies = [ [[package]] name = "sp-authorship" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -6822,7 +7014,7 @@ dependencies = [ [[package]] name = "sp-block-builder" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-api", @@ -6833,11 +7025,11 @@ dependencies = [ [[package]] name = "sp-blockchain" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "log 0.4.8", - "lru 0.4.3", + "lru", "parity-scale-codec", "parking_lot 0.10.0", "sp-block-builder", @@ -6848,12 +7040,12 @@ dependencies = [ [[package]] name = "sp-consensus" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "derive_more", "futures 0.3.4", "futures-diagnose", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -6870,7 +7062,7 @@ dependencies = [ [[package]] name = "sp-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-api", @@ -6883,7 +7075,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -6898,7 +7090,7 @@ dependencies = [ [[package]] name = "sp-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-api", @@ -6909,7 +7101,7 @@ dependencies = [ [[package]] name = "sp-core" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "base58", "blake2-rfc", @@ -6920,7 +7112,7 @@ dependencies = [ "hash256-std-hasher", "hex", "hex-literal", - "impl-serde 0.2.3", + "impl-serde 0.3.0", "lazy_static", "libsecp256k1", "log 0.4.8", @@ -6953,16 +7145,16 @@ dependencies = [ [[package]] name = "sp-debug-derive" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-externalities" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "environmental", "sp-std", @@ -6971,7 +7163,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "serde", @@ -6983,7 +7175,7 @@ dependencies = [ [[package]] name = "sp-finality-tracker" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -6992,7 +7184,7 @@ dependencies = [ [[package]] name = "sp-inherents" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "parity-scale-codec", @@ -7003,7 +7195,7 @@ dependencies = [ [[package]] name = "sp-io" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "hash-db", "libsecp256k1", @@ -7020,7 +7212,7 @@ dependencies = [ [[package]] name = "sp-keyring" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "lazy_static", "sp-core", @@ -7030,7 +7222,7 @@ dependencies = [ [[package]] name = "sp-offchain" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "sp-api", "sp-runtime", @@ -7038,7 +7230,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "backtrace", "log 0.4.8", @@ -7046,7 +7238,7 @@ dependencies = [ [[package]] name = "sp-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "rand 0.7.3", "serde", @@ -7058,7 +7250,7 @@ dependencies = [ [[package]] name = "sp-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "serde", "serde_json", @@ -7067,7 +7259,7 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-trait-for-tuples", "log 0.4.8", @@ -7087,7 +7279,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "primitive-types", @@ -7106,18 +7298,18 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "Inflector", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-runtime-interface-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sc-executor", "sp-core", @@ -7129,7 +7321,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-test-wasm" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-core", "sp-io", @@ -7140,7 +7332,7 @@ dependencies = [ [[package]] name = "sp-sandbox" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "assert_matches", "parity-scale-codec", @@ -7154,7 +7346,7 @@ dependencies = [ [[package]] name = "sp-serializer" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "serde", "serde_json", @@ -7162,7 +7354,7 @@ dependencies = [ [[package]] name = "sp-session" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "sp-api", "sp-core", @@ -7172,7 +7364,7 @@ dependencies = [ [[package]] name = "sp-staking" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -7181,7 +7373,7 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.8.0" +version = "0.8.0-alpha.1" dependencies = [ "hash-db", "hex-literal", @@ -7194,17 +7386,17 @@ dependencies = [ "sp-externalities", "sp-panic-handler", "sp-trie", - "trie-db 0.19.2", - "trie-root 0.15.2", + "trie-db", + "trie-root", ] [[package]] name = "sp-std" -version = "2.0.0" +version = "2.0.0-alpha.1" [[package]] name = "sp-storage" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-serde 0.2.3", "serde", @@ -7214,7 +7406,7 @@ dependencies = [ [[package]] name = "sp-test-primitives" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "parity-util-mem", @@ -7226,7 +7418,7 @@ dependencies = [ [[package]] name = "sp-timestamp" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7234,11 +7426,12 @@ dependencies = [ "sp-inherents", "sp-runtime", "sp-std", + "wasm-timer", ] [[package]] name = "sp-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "derive_more", "futures 0.3.4", @@ -7251,7 +7444,7 @@ dependencies = [ [[package]] name = "sp-trie" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "criterion 0.2.11", "hash-db", @@ -7261,14 +7454,14 @@ dependencies = [ "sp-core", "sp-std", "trie-bench", - "trie-db 0.20.0", - "trie-root 0.16.0", + "trie-db", + "trie-root", "trie-standardmap", ] [[package]] name = "sp-version" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-serde 0.2.3", "parity-scale-codec", @@ -7279,7 +7472,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7357,8 +7550,8 @@ checksum = "095064aa1f5b94d14e635d0a5684cf140c43ae40a0fd990708d38f5d669e5f64" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7378,14 +7571,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" dependencies = [ "heck", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "subkey" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "clap", "derive_more", @@ -7396,6 +7589,7 @@ dependencies = [ "hyper 0.12.35", "itertools", "jsonrpc-core-client", + "libp2p", "node-primitives", "node-runtime", "pallet-balances", @@ -7405,6 +7599,7 @@ dependencies = [ "rpassword", "rustc-hex", "sc-rpc", + "serde_json", "sp-core", "sp-runtime", "substrate-bip39", @@ -7425,11 +7620,11 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" -version = "2.0.0" +version = "2.0.0-alpha.1" [[package]] name = "substrate-frame-rpc-support" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "frame-support", "frame-system", @@ -7445,7 +7640,7 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" -version = "2.0.0" +version = "2.0.0-alpha.1" dependencies = [ "env_logger 0.7.1", "frame-system-rpc-runtime-api", @@ -7468,7 +7663,7 @@ dependencies = [ [[package]] name = "substrate-test-client" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "hash-db", @@ -7487,7 +7682,7 @@ dependencies = [ [[package]] name = "substrate-test-runtime" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "cfg-if", "frame-executive", @@ -7523,12 +7718,12 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "substrate-wasm-builder-runner", - "trie-db 0.20.0", + "trie-db", ] [[package]] name = "substrate-test-runtime-client" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "parity-scale-codec", @@ -7545,13 +7740,14 @@ dependencies = [ [[package]] name = "substrate-test-runtime-transaction-pool" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "derive_more", "futures 0.3.4", "parity-scale-codec", "parking_lot 0.10.0", "sc-transaction-graph", + "sp-blockchain", "sp-runtime", "sp-transaction-pool", "substrate-test-runtime-client", @@ -7559,7 +7755,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" -version = "2.0.0" +version = "2.0.0-alpha.1" [[package]] name = "substrate-wasm-builder" @@ -7593,13 +7789,13 @@ checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +checksum = "7a0294dc449adc58bb6592fff1a23d3e5e6e235afc6a0ffca2657d19e7bbffe5" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "unicode-xid 0.2.0", + "proc-macro2", + "quote", + "unicode-xid", ] [[package]] @@ -7608,8 +7804,8 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7619,10 +7815,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", - "unicode-xid 0.2.0", + "unicode-xid", ] [[package]] @@ -7646,9 +7842,9 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "target-lexicon" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4c118a7a38378f305a9e111fcb2f7f838c0be324bfb31a77ea04f7f6e684b4" +checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" [[package]] name = "target_info" @@ -7686,8 +7882,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9" dependencies = [ "lazy_static", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "version_check 0.9.1", ] @@ -7716,8 +7912,8 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57e4d2e50ca050ed44fb58309bdce3efa79948f84f9993ad1978de5eebdce5a7" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7752,16 +7948,16 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" +checksum = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92" dependencies = [ "failure", - "hashbrown 0.1.8", "hmac", - "once_cell 0.1.8", + "once_cell", "pbkdf2", - "rand 0.6.5", + "rand 0.7.3", + "rustc-hash", "sha2", ] @@ -7825,6 +8021,7 @@ checksum = "8fdd17989496f49cdc57978c96f0c9fe5e4a58a8bddc6813c449a4624f6a030b" dependencies = [ "bytes 0.5.4", "fnv", + "iovec", "lazy_static", "libc", "memchr", @@ -7833,6 +8030,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "signal-hook-registry", + "slab", "tokio-macros", "winapi 0.3.8", ] @@ -7918,8 +8116,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4b1e7ed7d5d4c2af3d999904b0eebe76544897cdbfb2b9684bed2174ab20f7c" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7944,15 +8142,13 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.10.3" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d7cf08f990090abd6c6a73cab46fed62f85e8aef8b99e4b918a9f4a637f0676" +checksum = "141afec0978abae6573065a48882c6bae44c5cc61db9b511ac4abf6a09bfd9cc" dependencies = [ - "bytes 0.4.12", - "futures 0.1.29", - "iovec", + "futures-core", "rustls", - "tokio-io", + "tokio 0.2.11", "webpki", ] @@ -8110,7 +8306,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cfd395def5a60236e187e1ff905cb55668a59f29928dec05e6e1b1fd2ac1f3" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -8146,25 +8342,11 @@ dependencies = [ "keccak-hasher", "memory-db", "parity-scale-codec", - "trie-db 0.20.0", - "trie-root 0.16.0", + "trie-db", + "trie-root", "trie-standardmap", ] -[[package]] -name = "trie-db" -version = "0.19.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d747ae5b6f078df7e46477fcc7df66df9eb4f27a031cf4a7c890a8dd03d8e6" -dependencies = [ - "hash-db", - "hashbrown 0.6.3", - "log 0.4.8", - "rand 0.6.5", - "rustc-hex", - "smallvec 1.2.0", -] - [[package]] name = "trie-db" version = "0.20.0" @@ -8172,21 +8354,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" dependencies = [ "hash-db", - "hashbrown 0.6.3", + "hashbrown", "log 0.4.8", "rustc-hex", "smallvec 1.2.0", ] -[[package]] -name = "trie-root" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" -dependencies = [ - "hash-db", -] - [[package]] name = "trie-root" version = "0.16.0" @@ -8214,9 +8387,9 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "trybuild" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f5b3f750c701725331ac78e389b5d143b7d25f6b6ffffd0d419759a9063ac5f" +checksum = "26ff1b18659a2218332848d76ad1c867ce4c6ee37b085e6bc8de9a6d11401220" dependencies = [ "glob 0.3.0", "lazy_static", @@ -8320,25 +8493,25 @@ checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" [[package]] name = "unicode-xid" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] -name = "unicode-xid" -version = "0.2.0" +name = "unsigned-varint" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "a7f0023a96687fe169081e8adce3f65e3874426b7886e9234d490af2dc077959" [[package]] name = "unsigned-varint" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c689459fbaeb50e56c6749275f084decfd02194ac5852e6617d95d0d3cf02eaf" +checksum = "3b7ffb36714206d2f5f05d61a2bc350415c642f2c54433f0ebf829afbe41d570" dependencies = [ "bytes 0.5.4", + "futures 0.3.4", "futures_codec", - "tokio-util", ] [[package]] @@ -8490,8 +8663,8 @@ dependencies = [ "bumpalo", "lazy_static", "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-shared", ] @@ -8514,7 +8687,7 @@ version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3" dependencies = [ - "quote 1.0.2", + "quote", "wasm-bindgen-macro-support", ] @@ -8524,8 +8697,8 @@ version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -8546,8 +8719,8 @@ dependencies = [ "anyhow", "heck", "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-backend", "weedle", @@ -8606,35 +8779,61 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.39.3" +version = "0.48.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "073da89bf1c84db000dd68ce660c1b4a08e3a2d28fd1e3394ab9e7abdde4a0f8" + +[[package]] +name = "wasmparser" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c702914acda5feeeffbc29e4d953e5b9ce79d8b98da4dbf18a77086e116c5470" +checksum = "9e41b27a1677fe28c115de49efca55dabb14f7fece2c32947ffb9b1064fe5bd4" + +[[package]] +name = "wasmtime" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614d964c3e7d07a13b59aca66103c52656bd80430f0d86dc7eeb3af4f03d4a2" +dependencies = [ + "anyhow", + "backtrace", + "cfg-if", + "lazy_static", + "libc", + "region", + "rustc-demangle", + "target-lexicon", + "wasmparser 0.51.1", + "wasmtime-environ", + "wasmtime-jit", + "wasmtime-runtime", + "wat", + "winapi 0.3.8", +] [[package]] name = "wasmtime-debug" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5008729ad53f75020f28fa0d682269335d6f0eac0b3ffafe31f185b2f33aca74" +checksum = "feb5900275b4ef0b621ce725b9d5660b12825d7f7d79b392b97baf089ffab8c0" dependencies = [ "anyhow", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", "faerie", - "gimli", + "gimli 0.19.0", "more-asserts", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.51.1", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3947662a0b8e05b1418465e64f16de9114f9fec18cc3f56e0ed5aa7737b89d0" +checksum = "f04661851e133fb11691c4a0f92a705766b4bbf7afc06811f949e295cc8414fc" dependencies = [ + "anyhow", "base64 0.11.0", "bincode", "cranelift-codegen", @@ -8644,37 +8843,37 @@ dependencies = [ "errno", "file-per-thread-logger", "indexmap", - "lazy_static", "libc", "log 0.4.8", "more-asserts", "rayon", "serde", "sha2", - "spin", "thiserror", "toml", - "wasmparser", + "wasmparser 0.51.1", "winapi 0.3.8", "zstd", ] [[package]] name = "wasmtime-jit" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed7922689461a7b5bd0d9c7350cac526c8a520a23b3ffd7f5b446ac51dfc51f" +checksum = "d451353764ce55c9bb6a8b260063cfc209b7adadd277a9a872ab4563a69e357c" dependencies = [ "anyhow", + "cfg-if", "cranelift-codegen", "cranelift-entity", "cranelift-frontend", + "cranelift-native", "cranelift-wasm", "more-asserts", "region", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.51.1", "wasmtime-debug", "wasmtime-environ", "wasmtime-runtime", @@ -8683,16 +8882,14 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "781d6bb8b346efaa3dc39746386957cd79b8d841e8652ed9b02d77bcf64fb514" +checksum = "7dbd4fc114b828cae3e405fed413df4b3814d87a92ea029640cec9ba41f0c162" dependencies = [ + "backtrace", "cc", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", + "cfg-if", "indexmap", - "lazy_static", "libc", "memoffset", "more-asserts", @@ -8702,6 +8899,24 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "wast" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a729d076deb29c8509fa71f2d427729f9394f9496844ed8fcab152f35d163d" +dependencies = [ + "leb128", +] + +[[package]] +name = "wat" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5795e34a4b39893653dec97e644fac85c31398e0ce1abecc48967aac83d9e8ce" +dependencies = [ + "wast", +] + [[package]] name = "web-sys" version = "0.3.35" @@ -8793,16 +9008,6 @@ dependencies = [ "nom", ] -[[package]] -name = "which" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" -dependencies = [ - "failure", - "libc", -] - [[package]] name = "which" version = "3.1.0" @@ -8894,6 +9099,17 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "x25519-dalek" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637ff90c9540fa3073bb577e65033069e4bae7c79d49d74aa3ffdf5342a53217" +dependencies = [ + "curve25519-dalek 2.0.0", + "rand_core 0.5.1", + "zeroize 1.1.0", +] + [[package]] name = "xdg" version = "2.2.0" @@ -8902,9 +9118,9 @@ checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" [[package]] name = "yamux" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902f4cee32c401c211b6b69f4a3f6f4cf3515644db5bd822cf685a7dbd6201f9" +checksum = "d73295bc9d9acf89dd9336b3b5f5b57731ee72b587857dd4312721a0196b48e5" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -8936,8 +9152,8 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index 47e3fe3f0efcee9c6fd02e5ff53a0af0369ccd04..1ac2beb052fac240127dbc4f394c714ac314e495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "client/cli", "client/consensus/aura", "client/consensus/babe", + "client/consensus/babe/rpc", "client/consensus/manual-seal", "client/consensus/pow", "client/consensus/uncles", @@ -35,6 +36,7 @@ members = [ "client/executor/wasmtime", "client/executor/runtime-test", "client/finality-grandpa", + "client/informant", "client/tracing", "client/keystore", "client/network", @@ -51,15 +53,15 @@ members = [ "client/telemetry", "client/transaction-pool", "client/transaction-pool/graph", + "utils/prometheus", "utils/wasm-builder-runner", - "utils/grafana-data-source", - "utils/grafana-data-source/test", "frame/assets", "frame/aura", "frame/authority-discovery", "frame/authorship", "frame/babe", "frame/balances", + "frame/benchmarking", "frame/collective", "frame/contracts", "frame/contracts/rpc", @@ -69,6 +71,7 @@ members = [ "frame/elections", "frame/evm", "frame/example", + "frame/example-offchain-worker", "frame/executive", "frame/finality-tracker", "frame/generic-asset", @@ -154,6 +157,7 @@ members = [ "utils/browser", "utils/build-script-utils", "utils/fork-tree", + "utils/frame/benchmarking-cli", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/wasm-builder", diff --git a/bin/node-template/README.md b/bin/node-template/README.md index c411dbeef5bcccf9e05d64ddee2405cb5237c678..4ae60478fc72b77f32664b4225c023c36ff5f1af 100644 --- a/bin/node-template/README.md +++ b/bin/node-template/README.md @@ -1,6 +1,6 @@ # Substrate Node Template -A new SRML-based Substrate node, ready for hacking. +A new FRAME-based Substrate node, ready for hacking. ## Build diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 9ad4a0e8a55ad5ad82a07252f1d62fc12e8ce101..4933b778db1fa89cc2d0dd69ba48c07209e838cc 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "node-template" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Anonymous"] edition = "2018" license = "Unlicense" build = "build.rs" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [[bin]] name = "node-template" @@ -14,25 +16,25 @@ futures = "0.3.1" log = "0.4.8" structopt = "0.3.8" -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sc-executor = { version = "0.8", path = "../../../client/executor" } -sc-service = { version = "0.8", path = "../../../client/service" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sc-transaction-pool = { version = "2.0.0", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8", path = "../../../client/network" } -sc-consensus-aura = { version = "0.8", path = "../../../client/consensus/aura" } -sp-consensus-aura = { version = "0.8", path = "../../../primitives/consensus/aura" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sc-client = { version = "0.8", path = "../../../client/" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sc-basic-authorship = { path = "../../../client/basic-authorship" } +sc-cli = { version = "0.8.0-alpha.1", path = "../../../client/cli" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sc-executor = { version = "0.8.0-alpha.1", path = "../../../client/executor" } +sc-service = { version = "0.8.0-alpha.1", path = "../../../client/service" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-alpha.1", path = "../../../client/network" } +sc-consensus-aura = { version = "0.8.0-alpha.1", path = "../../../client/consensus/aura" } +sp-consensus-aura = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/aura" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +grandpa = { version = "0.8.0-alpha.1", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +grandpa-primitives = { version = "2.0.0-alpha.1", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sc-client = { version = "0.8.0-alpha.1", path = "../../../client/" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sc-basic-authorship = { path = "../../../client/basic-authorship" , version = "0.8.0-alpha.1"} -node-template-runtime = { version = "2.0.0", path = "../runtime" } +node-template-runtime = { version = "2.0.0-alpha.1", path = "../runtime" } [build-dependencies] vergen = "3.0.4" -build-script-utils = { version = "2.0.0", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } +build-script-utils = { version = "2.0.0-alpha.1", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index aa50d06b2391675152b51eb120e723ffbeccd003..64b84005072fda218c5b866e8ec8a1898e27df18 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -1,7 +1,7 @@ use sp_core::{Pair, Public, sr25519}; use node_template_runtime::{ AccountId, AuraConfig, BalancesConfig, GenesisConfig, GrandpaConfig, - SudoConfig, IndicesConfig, SystemConfig, WASM_BINARY, Signature + SudoConfig, SystemConfig, WASM_BINARY, Signature }; use sp_consensus_aura::sr25519::{AuthorityId as AuraId}; use grandpa_primitives::{AuthorityId as GrandpaId}; @@ -127,21 +127,18 @@ fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, code: WASM_BINARY.to_vec(), changes_trie_config: Default::default(), }), - indices: Some(IndicesConfig { - ids: endowed_accounts.clone(), - }), balances: Some(BalancesConfig { balances: endowed_accounts.iter().cloned().map(|k|(k, 1 << 60)).collect(), }), - sudo: Some(SudoConfig { - key: root_key, - }), aura: Some(AuraConfig { authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(), }), grandpa: Some(GrandpaConfig { authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(), }), + sudo: Some(SudoConfig { + key: root_key, + }), } } diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e7e386703deee0a7894dfc5bf931c6c41fa89438..0f4c301dbff5b7defe0565bf71e85b21a6418867 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -15,32 +15,35 @@ // along with Substrate. If not, see . use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use sc_cli::{VersionInfo, error}; +use sc_cli::VersionInfo; use crate::service; use crate::chain_spec; use crate::cli::Cli; /// Parse and run command line arguments -pub fn run(version: VersionInfo) -> error::Result<()> { +pub fn run(version: VersionInfo) -> sc_cli::Result<()> { let opt = sc_cli::from_args::(&version); - let config = sc_service::Configuration::new(&version); + let mut config = sc_service::Configuration::from_version(&version); match opt.subcommand { - Some(subcommand) => sc_cli::run_subcommand( - config, - subcommand, - chain_spec::load_spec, - |config: _| Ok(new_full_start!(config).0), - &version, - ), - None => sc_cli::run( - config, - opt.run, - service::new_light, - service::new_full, - chain_spec::load_spec, - &version, - ) + Some(subcommand) => { + subcommand.init(&version)?; + subcommand.update_config(&mut config, chain_spec::load_spec, &version)?; + subcommand.run( + config, + |config: _| Ok(new_full_start!(config).0), + ) + }, + None => { + opt.run.init(&version)?; + opt.run.update_config(&mut config, chain_spec::load_spec, &version)?; + opt.run.run( + config, + service::new_light, + service::new_full, + &version, + ) + }, } } diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 9d0a57d77a851d551e9a8e087365a8d8cb470977..91b2c257e0cd733f7e87d00e02857101810a2fb0 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -7,10 +7,8 @@ mod service; mod cli; mod command; -pub use sc_cli::{VersionInfo, error}; - -fn main() -> Result<(), error::Error> { - let version = VersionInfo { +fn main() -> sc_cli::Result<()> { + let version = sc_cli::VersionInfo { name: "Substrate Node", commit: env!("VERGEN_SHA_SHORT"), version: env!("CARGO_PKG_VERSION"), diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index d113f90866a405e2ac062054b0a94ba75b98112b..6298f194e4f25a87d3a6ca12e32944d78ea092c2 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -6,7 +6,6 @@ use sc_client::LongestChain; use node_template_runtime::{self, GenesisConfig, opaque::Block, RuntimeApi}; use sc_service::{error::{Error as ServiceError}, AbstractService, Configuration, ServiceBuilder}; use sp_inherents::InherentDataProviders; -use sc_network::{construct_simple_protocol}; use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; @@ -19,11 +18,6 @@ native_executor_instance!( node_template_runtime::native_version, ); -construct_simple_protocol! { - /// Demo protocol attachment for substrate. - pub struct NodeProtocol where Block = Block { } -} - /// Starts a `ServiceBuilder` for a full service. /// /// Use this macro if you don't actually need the full service, but just the builder in order to @@ -41,10 +35,9 @@ macro_rules! new_full_start { })? .with_transaction_pool(|config, client, _fetcher| { let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); - let pool = sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api)); - Ok(pool) + Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api))) })? - .with_import_queue(|_config, client, mut select_chain, transaction_pool| { + .with_import_queue(|_config, client, mut select_chain, _transaction_pool| { let select_chain = select_chain.take() .ok_or_else(|| sc_service::Error::SelectChainRequired)?; @@ -57,14 +50,13 @@ macro_rules! new_full_start { grandpa_block_import.clone(), client.clone(), ); - let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, _>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair>( + sc_consensus_aura::slot_duration(&*client)?, aura_block_import, Some(Box::new(grandpa_block_import.clone())), None, client, inherent_data_providers.clone(), - Some(transaction_pool), )?; import_setup = Some((grandpa_block_import, grandpa_link)); @@ -96,7 +88,7 @@ pub fn new_full(config: Configuration) import_setup.take() .expect("Link Half and Block Import are present for Full Services or setup failed before. qed"); - let service = builder.with_network_protocol(|_| Ok(NodeProtocol::new()))? + let service = builder .with_finality_proof_provider(|client, backend| Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) )? @@ -116,7 +108,7 @@ pub fn new_full(config: Configuration) sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone()); let aura = sc_consensus_aura::start_aura::<_, _, _, _, _, AuraPair, _, _, _>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + sc_consensus_aura::slot_duration(&*client)?, client, select_chain, block_import, @@ -146,46 +138,41 @@ pub fn new_full(config: Configuration) gossip_duration: Duration::from_millis(333), justification_period: 512, name: Some(name), - observer_enabled: true, + observer_enabled: false, keystore, is_authority, }; - match (is_authority, disable_grandpa) { - (false, false) => { - // start the lightweight GRANDPA observer - service.spawn_task("grandpa-observer", grandpa::run_grandpa_observer( - grandpa_config, - grandpa_link, - service.network(), - service.on_exit(), - service.spawn_task_handle(), - )?); - }, - (true, false) => { - // start the full GRANDPA voter - let voter_config = grandpa::GrandpaParams { - config: grandpa_config, - link: grandpa_link, - network: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - on_exit: service.on_exit(), - telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), - executor: service.spawn_task_handle(), - }; - - // the GRANDPA voter task is considered infallible, i.e. - // if it fails we take down the service with it. - service.spawn_essential_task("grandpa", grandpa::run_grandpa_voter(voter_config)?); - }, - (_, true) => { - grandpa::setup_disabled_grandpa( - service.client(), - &inherent_data_providers, - service.network(), - )?; - }, + let enable_grandpa = !disable_grandpa; + if enable_grandpa { + // start the full GRANDPA voter + // NOTE: non-authorities could run the GRANDPA observer protocol, but at + // this point the full voter should provide better guarantees of block + // and vote data availability than the observer. The observer has not + // been tested extensively yet and having most nodes in a network run it + // could lead to finality stalls. + let grandpa_config = grandpa::GrandpaParams { + config: grandpa_config, + link: grandpa_link, + network: service.network(), + inherent_data_providers: inherent_data_providers.clone(), + on_exit: service.on_exit(), + telemetry_on_connect: Some(service.telemetry_on_connect_stream()), + voting_rule: grandpa::VotingRulesBuilder::default().build(), + }; + + // the GRANDPA voter task is considered infallible, i.e. + // if it fails we take down the service with it. + service.spawn_essential_task( + "grandpa-voter", + grandpa::run_grandpa_voter(grandpa_config)? + ); + } else { + grandpa::setup_disabled_grandpa( + service.client(), + &inherent_data_providers, + service.network(), + )?; } Ok(service) @@ -222,19 +209,17 @@ pub fn new_light(config: Configuration) let finality_proof_request_builder = finality_proof_import.create_finality_proof_request_builder(); - let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, ()>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair>( + sc_consensus_aura::slot_duration(&*client)?, grandpa_block_import, None, Some(Box::new(finality_proof_import)), client, inherent_data_providers.clone(), - None, )?; Ok((import_queue, finality_proof_request_builder)) })? - .with_network_protocol(|_| Ok(NodeProtocol::new()))? .with_finality_proof_provider(|client, backend| Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) )? diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 8ea3f3adabc52c2c87802b2f407d188f14bd202c..bd08787f1aa706b253c7b9a98e1a2e6fa76209d4 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -2,7 +2,9 @@ authors = ['Anonymous'] edition = '2018' name = 'pallet-template' -version = '2.0.0' +version = "2.0.0-alpha.1" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } @@ -10,28 +12,28 @@ safe-mix = { default-features = false, version = '1.0.0' } [dependencies.frame-support] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.1" path = "../../../../frame/support" [dependencies.system] default-features = false package = 'frame-system' -version = '2.0.0' +version = "2.0.0-alpha.1" path = "../../../../frame/system" [dev-dependencies.sp-core] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.1" path = "../../../../primitives/core" [dev-dependencies.sp-io] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.1" path = "../../../../primitives/io" [dev-dependencies.sp-runtime] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.1" path = "../../../../primitives/runtime" [features] diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index a1615b4c1f0bc071b7cf54e3977284df8d411061..34e055bf546b0b57a247a8cb083a41de7d064508 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -1,12 +1,12 @@ #![cfg_attr(not(feature = "std"), no_std)] -/// A runtime module template with necessary imports +/// A FRAME pallet template with necessary imports /// Feel free to remove or edit this file as needed. /// If you change the name of this file, make sure to update its references in runtime/src/lib.rs /// If you remove this file, you can remove those references -/// For more guidance on Substrate modules, see the example module +/// For more guidance on Substrate FRAME, see the example pallet /// https://github.com/paritytech/substrate/blob/master/frame/example/src/lib.rs use frame_support::{decl_module, decl_storage, decl_event, decl_error, dispatch}; @@ -41,7 +41,7 @@ decl_event!( pub enum Event where AccountId = ::AccountId { /// Just a dummy event. /// Event `Something` is declared with a parameter of the type `u32` and `AccountId` - /// To emit this event, we call the deposit funtion, from our runtime funtions + /// To emit this event, we call the deposit function, from our runtime functions SomethingStored(u32, AccountId), } ); diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index 2cbfc89d5b3ae69e3737b2667a3b7972effcdf70..2ea81ffb456261122b234c6ae3c36f52eaa9c430 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -11,9 +11,9 @@ impl_outer_origin! { pub enum Origin for Test {} } -// For testing the module, we construct most of a mock runtime. This means +// For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the -// configuration traits of modules we want to use. +// configuration traits of pallets we want to use. #[derive(Clone, Eq, PartialEq)] pub struct Test; parameter_types! { @@ -39,6 +39,9 @@ impl system::Trait for Test { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Event = (); diff --git a/bin/node-template/pallets/template/src/tests.rs b/bin/node-template/pallets/template/src/tests.rs index 44a423c948fbf88cb193612fbab307d4df20b634..ec123a50c7cc984f15c8f80983ed300d95ecdccf 100644 --- a/bin/node-template/pallets/template/src/tests.rs +++ b/bin/node-template/pallets/template/src/tests.rs @@ -6,7 +6,7 @@ use frame_support::{assert_ok, assert_noop}; #[test] fn it_works_for_default_value() { new_test_ext().execute_with(|| { - // Just a dummy test for the dummy funtion `do_something` + // Just a dummy test for the dummy function `do_something` // calling the `do_something` function with a value 42 assert_ok!(TemplateModule::do_something(Origin::signed(1), 42)); // asserting that the stored value is equal to what we stored diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index ddecb0e4cff4c11da2ddf6e7973c1c57e082d1e1..049953eceadc5bb5bd1d3374cf6168e35117d64c 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -1,42 +1,43 @@ [package] name = "node-template-runtime" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Anonymous"] edition = "2018" license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -aura = { version = "2.0.0", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } -balances = { version = "2.0.0", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -grandpa = { version = "2.0.0", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } -indices = { version = "2.0.0", default-features = false, package = "pallet-indices", path = "../../../frame/indices" } -randomness-collective-flip = { version = "2.0.0", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } -sudo = { version = "2.0.0", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } -system = { version = "2.0.0", default-features = false, package = "frame-system", path = "../../../frame/system" } -timestamp = { version = "2.0.0", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } -transaction-payment = { version = "2.0.0", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } -frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } +aura = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } +balances = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/support" } +grandpa = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } +randomness-collective-flip = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } +sudo = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } +system = { version = "2.0.0-alpha.1", default-features = false, package = "frame-system", path = "../../../frame/system" } +timestamp = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } +transaction-payment = { version = "2.0.0-alpha.1", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } +frame-executive = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/executive" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0", default-features = false, path = "../../../primitives/api" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false} -sp-consensus-aura = { version = "0.8", default-features = false, path = "../../../primitives/consensus/aura" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-inherents = { path = "../../../primitives/inherents", default-features = false} -sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" } -sp-offchain = { version = "2.0.0", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } +sp-api = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/api" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.1"} +sp-consensus-aura = { version = "0.8.0-alpha.1", default-features = false, path = "../../../primitives/consensus/aura" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/core" } +sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-alpha.1"} +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/io" } +sp-offchain = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/offchain" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/runtime" } +sp-session = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/session" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/std" } +sp-transaction-pool = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/version" } -template = { version = "2.0.0", default-features = false, path = "../pallets/template", package = "pallet-template" } +template = { version = "2.0.0-alpha.1", default-features = false, path = "../pallets/template", package = "pallet-template" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [features] default = ["std"] @@ -47,7 +48,6 @@ std = [ "frame-executive/std", "frame-support/std", "grandpa/std", - "indices/std", "randomness-collective-flip/std", "serde", "sp-api/std", diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index a863ec40a70a9ff7dca85cd10f09f5ff5ae57baa..2bc4c2745007beb069900c48220efc8fb6f94569 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -12,10 +12,10 @@ use sp_std::prelude::*; use sp_core::OpaqueMetadata; use sp_runtime::{ ApplyExtrinsicResult, transaction_validity::TransactionValidity, generic, create_runtime_str, - impl_opaque_keys, MultiSignature + impl_opaque_keys, MultiSignature, }; use sp_runtime::traits::{ - BlakeTwo256, Block as BlockT, StaticLookup, Verify, ConvertInto, IdentifyAccount + BlakeTwo256, Block as BlockT, IdentityLookup, Verify, ConvertInto, IdentifyAccount }; use sp_api::impl_runtime_apis; use sp_consensus_aura::sr25519::AuthorityId as AuraId; @@ -69,7 +69,7 @@ pub type DigestItem = generic::DigestItem; /// Opaque types. These are used by the CLI to instantiate machinery that don't need to know /// the specifics of the runtime. They can then be made to be agnostic over specific formats /// of data like extrinsics, allowing for them to continue syncing the network through upgrades -/// to even the core datastructures. +/// to even the core data structures. pub mod opaque { use super::*; @@ -109,7 +109,7 @@ pub const MINUTES: BlockNumber = 60_000 / (MILLISECS_PER_BLOCK as BlockNumber); pub const HOURS: BlockNumber = MINUTES * 60; pub const DAYS: BlockNumber = HOURS * 24; -/// The version infromation used to identify this runtime when compiled natively. +/// The version information used to identify this runtime when compiled natively. #[cfg(feature = "std")] pub fn native_version() -> NativeVersion { NativeVersion { @@ -132,7 +132,7 @@ impl system::Trait for Runtime { /// The aggregated dispatch type that is available for extrinsics. type Call = Call; /// The lookup mechanism to get account ID from whatever is passed in dispatchers. - type Lookup = Indices; + type Lookup = IdentityLookup; /// The index type for storing how many extrinsics an account has signed. type Index = Index; /// The index type for blocks. @@ -161,6 +161,12 @@ impl system::Trait for Runtime { /// /// This type is being generated by `construct_runtime!`. type ModuleToIndex = ModuleToIndex; + /// What to do if a new account is created. + type OnNewAccount = (); + /// What to do if an account is fully reaped from the system. + type OnKilledAccount = (); + /// The data to be stored in an account. + type AccountData = balances::AccountData; } impl aura::Trait for Runtime { @@ -171,18 +177,6 @@ impl grandpa::Trait for Runtime { type Event = Event; } -impl indices::Trait for Runtime { - /// The type for recording indexing into the account enumeration. If this ever overflows, there - /// will be problems! - type AccountIndex = AccountIndex; - /// Use the standard means of resolving an index hint from an id. - type ResolveHint = indices::SimpleResolveHint; - /// Determine whether an account is dead. - type IsDeadAccount = Balances; - /// The ubiquitous event type. - type Event = Event; -} - parameter_types! { pub const MinimumPeriod: u64 = SLOT_DURATION / 2; } @@ -196,22 +190,16 @@ impl timestamp::Trait for Runtime { parameter_types! { pub const ExistentialDeposit: u128 = 500; - pub const CreationFee: u128 = 0; } impl balances::Trait for Runtime { /// The type for recording an account's balance. type Balance = Balance; - /// What to do if an account is fully reaped from the system. - type OnReapAccount = System; - /// What to do if a new account is created. - type OnNewAccount = Indices; /// The ubiquitous event type. type Event = Event; type DustRemoval = (); - type TransferPayment = (); type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = System; } parameter_types! { @@ -230,7 +218,7 @@ impl transaction_payment::Trait for Runtime { impl sudo::Trait for Runtime { type Event = Event; - type Proposal = Call; + type Call = Call; } /// Used for the module template in `./template.rs` @@ -244,22 +232,21 @@ construct_runtime!( NodeBlock = opaque::Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Module, Call, Storage, Config, Event}, + System: system::{Module, Call, Config, Storage, Event}, + RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, Timestamp: timestamp::{Module, Call, Storage, Inherent}, Aura: aura::{Module, Config, Inherent(Timestamp)}, Grandpa: grandpa::{Module, Call, Storage, Config, Event}, - Indices: indices, - Balances: balances, + Balances: balances::{Module, Call, Storage, Config, Event}, TransactionPayment: transaction_payment::{Module, Storage}, - Sudo: sudo, + Sudo: sudo::{Module, Call, Config, Storage, Event}, // Used for the module template in `./template.rs` TemplateModule: template::{Module, Call, Storage, Event}, - RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, } ); /// The address format for describing accounts. -pub type Address = ::Source; +pub type Address = AccountId; /// Block header type as expected by this runtime. pub type Header = generic::Header; /// Block type as expected by this runtime. @@ -310,6 +297,10 @@ impl_runtime_apis! { Executive::apply_extrinsic(extrinsic) } + fn apply_trusted_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResult { + Executive::apply_trusted_extrinsic(extrinsic) + } + fn finalize_block() -> ::Header { Executive::finalize_block() } diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index dad76ae4bfdad4b9f267f077bc168b8f1ed490ad..1aa49414bdd3c09846130d9f135b1616ae2c3fcf 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -1,12 +1,19 @@ [package] name = "node-cli" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." build = "build.rs" edition = "2018" license = "GPL-3.0" default-run = "substrate" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[package.metadata.wasm-pack.profile.release] +# `wasm-opt` has some problems on linux, see +# https://github.com/rustwasm/wasm-pack/issues/781 etc. +wasm-opt = false [badges] travis-ci = { repository = "paritytech/substrate", branch = "master" } @@ -35,79 +42,84 @@ structopt = { version = "0.3.8", optional = true } tracing = "0.1.10" # primitives -sp-authority-discovery = { version = "2.0.0", path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../../primitives/timestamp" } -sp-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../primitives/finality-tracker" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-authority-discovery = { version = "2.0.0-alpha.1", path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/babe" } +grandpa-primitives = { version = "2.0.0-alpha.1", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/timestamp" } +sp-finality-tracker = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/finality-tracker" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../../primitives/keyring" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } # client dependencies -sc-client-api = { version = "2.0.0", path = "../../../client/api" } -sc-client = { version = "0.8", path = "../../../client/" } -sc-chain-spec = { version = "2.0.0", path = "../../../client/chain-spec" } -sc-transaction-pool = { version = "2.0.0", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8", path = "../../../client/network" } -sc-consensus-babe = { version = "0.8", path = "../../../client/consensus/babe" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -sc-client-db = { version = "0.8", default-features = false, path = "../../../client/db" } -sc-offchain = { version = "2.0.0", path = "../../../client/offchain" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } -sc-basic-authorship = { version = "0.8", path = "../../../client/basic-authorship" } -sc-service = { version = "0.8", default-features = false, path = "../../../client/service" } -sc-tracing = { version = "2.0.0", path = "../../../client/tracing" } -sc-telemetry = { version = "2.0.0", path = "../../../client/telemetry" } -sc-authority-discovery = { version = "0.8", path = "../../../client/authority-discovery" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../../client/api" } +sc-client = { version = "0.8.0-alpha.1", path = "../../../client/" } +sc-chain-spec = { version = "2.0.0-alpha.1", path = "../../../client/chain-spec" } +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-alpha.1", path = "../../../client/network" } +sc-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../client/consensus/babe" } +grandpa = { version = "0.8.0-alpha.1", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +sc-client-db = { version = "0.8.0-alpha.1", default-features = false, path = "../../../client/db" } +sc-offchain = { version = "2.0.0-alpha.1", path = "../../../client/offchain" } +sc-rpc = { version = "2.0.0-alpha.1", path = "../../../client/rpc" } +sc-basic-authorship = { version = "0.8.0-alpha.1", path = "../../../client/basic-authorship" } +sc-service = { version = "0.8.0-alpha.1", default-features = false, path = "../../../client/service" } +sc-tracing = { version = "2.0.0-alpha.1", path = "../../../client/tracing" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../../../client/telemetry" } +sc-authority-discovery = { version = "0.8.0-alpha.1", path = "../../../client/authority-discovery" } # frame dependencies -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../../../frame/timestamp" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -pallet-im-online = { version = "2.0.0", default-features = false, path = "../../../frame/im-online" } -pallet-authority-discovery = { version = "2.0.0", path = "../../../frame/authority-discovery" } +pallet-indices = { version = "2.0.0-alpha.1", path = "../../../frame/indices" } +pallet-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/timestamp" } +pallet-contracts = { version = "2.0.0-alpha.1", path = "../../../frame/contracts" } +frame-system = { version = "2.0.0-alpha.1", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", path = "../../../frame/transaction-payment" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/support" } +pallet-im-online = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/im-online" } +pallet-authority-discovery = { version = "2.0.0-alpha.1", path = "../../../frame/authority-discovery" } # node-specific dependencies -node-runtime = { version = "2.0.0", path = "../runtime" } -node-rpc = { version = "2.0.0", path = "../rpc" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-executor = { version = "2.0.0", path = "../executor" } +node-runtime = { version = "2.0.0-alpha.1", path = "../runtime" } +node-rpc = { version = "2.0.0-alpha.1", path = "../rpc" } +node-primitives = { version = "2.0.0-alpha.1", path = "../primitives" } +node-executor = { version = "2.0.0-alpha.1", path = "../executor" } # CLI-specific dependencies -sc-cli = { version = "0.8.0", optional = true, path = "../../../client/cli" } -node-transaction-factory = { version = "0.8.0", optional = true, path = "../transaction-factory" } +sc-cli = { version = "0.8.0-alpha.1", optional = true, path = "../../../client/cli" } +frame-benchmarking-cli = { version = "2.0.0-alpha.1", optional = true, path = "../../../utils/frame/benchmarking-cli" } +node-transaction-factory = { version = "0.8.0-alpha.1", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-alpha.1", optional = true, path = "../inspect" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } wasm-bindgen-futures = { version = "0.4.7", optional = true } -browser-utils = { path = "../../../utils/browser", optional = true } +browser-utils = { path = "../../../utils/browser", optional = true , version = "0.8.0-alpha.1"} [dev-dependencies] -sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } -sc-consensus-babe = { version = "0.8", features = ["test-helpers"], path = "../../../client/consensus/babe" } -sc-consensus-epochs = { version = "0.8", path = "../../../client/consensus/epochs" } -sc-service-test = { version = "2.0.0", path = "../../../client/service/test" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../../client/keystore" } +sc-consensus-babe = { version = "0.8.0-alpha.1", features = ["test-helpers"], path = "../../../client/consensus/babe" } +sc-consensus-epochs = { version = "0.8.0-alpha.1", path = "../../../client/consensus/epochs" } +sc-service-test = { version = "2.0.0-dev", path = "../../../client/service/test" } futures = "0.3.1" tempfile = "3.1.0" assert_cmd = "0.12" nix = "0.17" +serde_json = "1.0" [build-dependencies] -build-script-utils = { version = "2.0.0", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } +build-script-utils = { version = "2.0.0-alpha.1", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } structopt = { version = "0.3.8", optional = true } -node-transaction-factory = { version = "0.8.0", optional = true, path = "../transaction-factory" } +node-transaction-factory = { version = "0.8.0-alpha.1", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-alpha.1", optional = true, path = "../inspect" } +frame-benchmarking-cli = { version = "2.0.0-alpha.1", optional = true, path = "../../../utils/frame/benchmarking-cli" } [build-dependencies.sc-cli] -version = "0.8.0" +version = "0.8.0-alpha.1" package = "sc-cli" path = "../../../client/cli" optional = true @@ -124,12 +136,14 @@ browser = [ "wasm-bindgen-futures", ] cli = [ - "sc-cli", + "node-executor/wasmi-errno", + "node-inspect", "node-transaction-factory", + "sc-cli", + "frame-benchmarking-cli", "sc-service/rocksdb", - "node-executor/wasmi-errno", - "vergen", "structopt", + "vergen", ] wasmtime = [ "cli", diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index e951c04710b92ff387b2f5fd5ba3c144b68af05c..8c4412667baceec56904864413178cbc88001497 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -18,10 +18,8 @@ #![warn(missing_docs)] -use sc_cli::VersionInfo; - -fn main() -> Result<(), sc_cli::error::Error> { - let version = VersionInfo { +fn main() -> sc_cli::Result<()> { + let version = sc_cli::VersionInfo { name: "Substrate Node", commit: env!("VERGEN_SHA_SHORT"), version: env!("CARGO_PKG_VERSION"), diff --git a/bin/node/cli/browser-demo/index.html b/bin/node/cli/browser-demo/index.html index 0b66b612f10f78c7db40b582bb2d8d2ed5eb73af..f40863c46e77d2d811848aa6f459d5fe8381b6fe 100644 --- a/bin/node/cli/browser-demo/index.html +++ b/bin/node/cli/browser-demo/index.html @@ -15,11 +15,12 @@ function log(msg) { async function start() { log('Loading WASM'); await init('./pkg/node_cli_bg.wasm'); - log('Successfully loaded WASM'); + log('Fetching chain spec'); + const chain_spec_response = await fetch("https://raw.githubusercontent.com/paritytech/substrate/master/bin/node/cli/res/flaming-fir.json"); + const chain_spec_text = await chain_spec_response.text(); // Build our client. - log('Starting client'); - let client = await start_client(ws()); + let client = await start_client(chain_spec_text, 'info', ws()); log('Client started'); client.rpcSubscribe('{"method":"chain_subscribeNewHead","params":[],"id":1,"jsonrpc":"2.0"}', diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index e05238b82ddf19df1785070dd87587dadbb19586..80ca963445d0c10b579f05e7ffe1a46b8a4ae41b 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::ChainSpec; +use crate::chain_spec::ChainSpec; use log::info; use wasm_bindgen::prelude::*; use sc_service::Configuration; @@ -22,20 +22,20 @@ use browser_utils::{ Transport, Client, browser_configuration, set_console_error_panic_hook, init_console_log, }; +use std::str::FromStr; /// Starts the client. #[wasm_bindgen] -pub async fn start_client(wasm_ext: Transport) -> Result { - start_inner(wasm_ext) +pub async fn start_client(chain_spec: String, log_level: String, wasm_ext: Transport) -> Result { + start_inner(chain_spec, log_level, wasm_ext) .await .map_err(|err| JsValue::from_str(&err.to_string())) } -async fn start_inner(wasm_ext: Transport) -> Result> { +async fn start_inner(chain_spec: String, log_level: String, wasm_ext: Transport) -> Result> { set_console_error_panic_hook(); - init_console_log(log::Level::Info)?; - - let chain_spec = ChainSpec::FlamingFir.load() + init_console_log(log::Level::from_str(&log_level)?)?; + let chain_spec = ChainSpec::from_json_bytes(chain_spec.as_bytes().to_vec()) .map_err(|e| format!("{:?}", e))?; let config: Configuration<_, _> = browser_configuration(wasm_ext, chain_spec) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 868b991480b89d4b916c5998c0562aaa1c489485..d163cdf3918a2847b84a8a54a40ee688e53f43e1 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -21,8 +21,8 @@ use sp_core::{Pair, Public, crypto::UncheckedInto, sr25519}; use serde::{Serialize, Deserialize}; use node_runtime::{ AuthorityDiscoveryConfig, BabeConfig, BalancesConfig, ContractsConfig, CouncilConfig, DemocracyConfig, - GrandpaConfig, ImOnlineConfig, IndicesConfig, SessionConfig, SessionKeys, StakerStatus, StakingConfig, - SocietyConfig, SudoConfig, SystemConfig, TechnicalCommitteeConfig, WASM_BINARY, + GrandpaConfig, ImOnlineConfig, SessionConfig, SessionKeys, StakerStatus, StakingConfig, + IndicesConfig, SocietyConfig, SudoConfig, SystemConfig, TechnicalCommitteeConfig, WASM_BINARY, }; use node_runtime::Block; use node_runtime::constants::currency::*; @@ -239,13 +239,11 @@ pub fn testnet_genesis( .collect(), }), pallet_indices: Some(IndicesConfig { - ids: endowed_accounts.iter().cloned() - .chain(initial_authorities.iter().map(|x| x.0.clone())) - .collect::>(), + indices: vec![], }), pallet_session: Some(SessionConfig { keys: initial_authorities.iter().map(|x| { - (x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone())) + (x.0.clone(), x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone())) }).collect::>(), }), pallet_staking: Some(StakingConfig { @@ -261,13 +259,17 @@ pub fn testnet_genesis( }), pallet_democracy: Some(DemocracyConfig::default()), pallet_collective_Instance1: Some(CouncilConfig { - members: endowed_accounts.iter().cloned() - .collect::>()[..(num_endowed_accounts + 1) / 2].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), phantom: Default::default(), }), pallet_collective_Instance2: Some(TechnicalCommitteeConfig { - members: endowed_accounts.iter().cloned() - .collect::>()[..(num_endowed_accounts + 1) / 2].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), phantom: Default::default(), }), pallet_contracts: Some(ContractsConfig { @@ -295,7 +297,10 @@ pub fn testnet_genesis( pallet_membership_Instance1: Some(Default::default()), pallet_treasury: Some(Default::default()), pallet_society: Some(SocietyConfig { - members: endowed_accounts[0..3].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), pot: 0, max_members: 999, }), @@ -359,6 +364,7 @@ pub(crate) mod tests { use super::*; use crate::service::{new_full, new_light}; use sc_service_test; + use sp_runtime::BuildStorage; fn local_testnet_genesis_instant_single() -> GenesisConfig { testnet_genesis( @@ -408,4 +414,19 @@ pub(crate) mod tests { |config| new_light(config), ); } + + #[test] + fn test_create_development_chain_spec() { + development_config().build_storage().unwrap(); + } + + #[test] + fn test_create_local_testnet_chain_spec() { + local_testnet_config().build_storage().unwrap(); + } + + #[test] + fn test_staging_test_net_chain_spec() { + staging_testnet_config().build_storage().unwrap(); + } } diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index dd3ee4a596605859fa332ea898ffe670a05e5d08..b6db9c3deb7e31f8f8da62d30b6971bf572b30d1 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -17,15 +17,10 @@ use sc_cli::{SharedParams, ImportParams, RunCmd}; use structopt::StructOpt; -#[allow(missing_docs)] +/// An overarching CLI command definition. #[derive(Clone, Debug, StructOpt)] -#[structopt(settings = &[ - structopt::clap::AppSettings::GlobalVersion, - structopt::clap::AppSettings::ArgsNegateSubcommands, - structopt::clap::AppSettings::SubcommandsNegateReqs, -])] pub struct Cli { - #[allow(missing_docs)] + /// Possible subcommand with parameters. #[structopt(subcommand)] pub subcommand: Option, #[allow(missing_docs)] @@ -33,10 +28,10 @@ pub struct Cli { pub run: RunCmd, } -#[allow(missing_docs)] +/// Possible subcommands of the main binary. #[derive(Clone, Debug, StructOpt)] pub enum Subcommand { - #[allow(missing_docs)] + /// A set of base subcommands handled by `sc_cli`. #[structopt(flatten)] Base(sc_cli::Subcommand), /// The custom factory subcommmand for manufacturing transactions. @@ -46,40 +41,33 @@ pub enum Subcommand { Only supported for development or local testnet." )] Factory(FactoryCmd), + + /// The custom inspect subcommmand for decoding blocks and extrinsics. + #[structopt( + name = "inspect", + about = "Decode given block or extrinsic using current native runtime." + )] + Inspect(node_inspect::cli::InspectCmd), + + /// The custom benchmark subcommmand benchmarking runtime pallets. + #[structopt( + name = "benchmark", + about = "Benchmark runtime pallets." + )] + Benchmark(frame_benchmarking_cli::BenchmarkCmd), } /// The `factory` command used to generate transactions. /// Please note: this command currently only works on an empty database! #[derive(Debug, StructOpt, Clone)] pub struct FactoryCmd { - /// How often to repeat. This option only has an effect in mode `MasterToNToM`. - #[structopt(long="rounds", default_value = "1")] - pub rounds: u64, - - /// MasterToN: Manufacture `num` transactions from the master account - /// to `num` randomly created accounts, one each. - /// - /// MasterTo1: Manufacture `num` transactions from the master account - /// to exactly one other randomly created account. - /// - /// MasterToNToM: Manufacture `num` transactions from the master account - /// to `num` randomly created accounts. - /// From each of these randomly created accounts manufacture - /// a transaction to another randomly created account. - /// Repeat this `rounds` times. If `rounds` = 1 the behavior - /// is the same as `MasterToN`.{n} - /// A -> B, A -> C, A -> D, ... x `num`{n} - /// B -> E, C -> F, D -> G, ...{n} - /// ... x `rounds` - /// - /// These three modes control manufacturing. - #[structopt(long="mode", default_value = "MasterToN")] - pub mode: node_transaction_factory::Mode, + /// Number of blocks to generate. + #[structopt(long="blocks", default_value = "1")] + pub blocks: u32, - /// Number of transactions to generate. In mode `MasterNToNToM` this is - /// the number of transactions per round. - #[structopt(long="num", default_value = "8")] - pub num: u64, + /// Number of transactions to push per block. + #[structopt(long="transactions", default_value = "8")] + pub transactions: u32, #[allow(missing_docs)] #[structopt(flatten)] diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 3bfed148f585599638ae284a92d6381da1a5d785..dfdf5533f2b343e5f3fbe372a4b3d058f9c97f07 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -14,13 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::{VersionInfo, error}; +use sc_cli::VersionInfo; use sc_service::{Roles as ServiceRoles}; use node_transaction_factory::RuntimeAdapter; use crate::{Cli, service, ChainSpec, load_spec, Subcommand, factory_impl::FactoryState}; /// Parse command line arguments into service configuration. -pub fn run(args: I, version: VersionInfo) -> error::Result<()> +pub fn run(args: I, version: VersionInfo) -> sc_cli::Result<()> where I: Iterator, T: Into + Clone, @@ -28,32 +28,52 @@ where let args: Vec<_> = args.collect(); let opt = sc_cli::from_iter::(args.clone(), &version); - let mut config = sc_service::Configuration::new(&version); + let mut config = sc_service::Configuration::from_version(&version); match opt.subcommand { - None => sc_cli::run( - config, - opt.run, - service::new_light, - service::new_full, - load_spec, - &version, - ), + None => { + opt.run.init(&version)?; + opt.run.update_config(&mut config, load_spec, &version)?; + opt.run.run( + config, + service::new_light, + service::new_full, + &version, + ) + }, + Some(Subcommand::Inspect(cmd)) => { + cmd.init(&version)?; + cmd.update_config(&mut config, load_spec, &version)?; + + let client = sc_service::new_full_client::< + node_runtime::Block, node_runtime::RuntimeApi, node_executor::Executor, _, _, + >(&config)?; + let inspect = node_inspect::Inspector::::new(client); + + cmd.run(inspect) + }, + Some(Subcommand::Benchmark(cmd)) => { + cmd.init(&version)?; + cmd.update_config(&mut config, load_spec, &version)?; + + cmd.run::<_, _, node_runtime::Block, node_executor::Executor>(config) + }, Some(Subcommand::Factory(cli_args)) => { - sc_cli::init(&cli_args.shared_params, &version)?; - sc_cli::init_config(&mut config, &cli_args.shared_params, &version, load_spec)?; - sc_cli::fill_import_params( + cli_args.shared_params.init(&version)?; + cli_args.shared_params.update_config(&mut config, load_spec, &version)?; + cli_args.import_params.update_config( &mut config, - &cli_args.import_params, ServiceRoles::FULL, cli_args.shared_params.dev, )?; - sc_cli::fill_config_keystore_in_memory(&mut config)?; + config.use_in_memory_keystore()?; match ChainSpec::from(config.expect_chain_spec().id()) { Some(ref c) if c == &ChainSpec::Development || c == &ChainSpec::LocalTestnet => {}, - _ => panic!("Factory is only supported for development and local testnet."), + _ => return Err( + "Factory is only supported for development and local testnet.".into() + ), } // Setup tracing. @@ -62,14 +82,15 @@ where cli_args.import_params.tracing_receiver.into(), tracing_targets ); if let Err(e) = tracing::subscriber::set_global_default(subscriber) { - panic!("Unable to set global default subscriber {}", e); + return Err( + format!("Unable to set global default subscriber {}", e).into() + ); } } let factory_state = FactoryState::new( - cli_args.mode.clone(), - cli_args.num, - cli_args.rounds, + cli_args.blocks, + cli_args.transactions, ); let service_builder = new_full_start!(config).0; @@ -82,12 +103,13 @@ where Ok(()) }, - Some(Subcommand::Base(subcommand)) => sc_cli::run_subcommand( - config, - subcommand, - load_spec, - |config: service::NodeConfiguration| Ok(new_full_start!(config).0), - &version, - ), + Some(Subcommand::Base(subcommand)) => { + subcommand.init(&version)?; + subcommand.update_config(&mut config, load_spec, &version)?; + subcommand.run( + config, + |config: service::NodeConfiguration| Ok(new_full_start!(config).0), + ) + }, } } diff --git a/bin/node/cli/src/factory_impl.rs b/bin/node/cli/src/factory_impl.rs index a1c5a5f4e050f5dbf8db06000f2d691225ab6eca..1d1eabe29cbfb12ca43663be00b0969e852a30d9 100644 --- a/bin/node/cli/src/factory_impl.rs +++ b/bin/node/cli/src/factory_impl.rs @@ -33,7 +33,6 @@ use sp_runtime::{ generic::Era, traits::{Block as BlockT, Header as HeaderT, SignedExtension, Verify, IdentifyAccount} }; use node_transaction_factory::RuntimeAdapter; -use node_transaction_factory::modes::Mode; use sp_inherents::InherentData; use sp_timestamp; use sp_finality_tracker; @@ -41,14 +40,10 @@ use sp_finality_tracker; type AccountPublic = ::Signer; pub struct FactoryState { - block_no: N, - - mode: Mode, - start_number: u32, - rounds: u32, - round: u32, - block_in_round: u32, - num: u32, + blocks: u32, + transactions: u32, + block_number: N, + index: u32, } type Number = <::Header as HeaderT>::Number; @@ -78,63 +73,35 @@ impl RuntimeAdapter for FactoryState { type Number = Number; fn new( - mode: Mode, - num: u64, - rounds: u64, + blocks: u32, + transactions: u32, ) -> FactoryState { FactoryState { - mode, - num: num as u32, - round: 0, - rounds: rounds as u32, - block_in_round: 0, - block_no: 0, - start_number: 0, + blocks, + transactions, + block_number: 0, + index: 0, } } - fn block_no(&self) -> Self::Number { - self.block_no - } - - fn block_in_round(&self) -> Self::Number { - self.block_in_round - } - - fn rounds(&self) -> Self::Number { - self.rounds - } - - fn num(&self) -> Self::Number { - self.num - } - - fn round(&self) -> Self::Number { - self.round - } - - fn start_number(&self) -> Self::Number { - self.start_number - } - - fn mode(&self) -> &Mode { - &self.mode + fn block_number(&self) -> u32 { + self.block_number } - fn set_block_no(&mut self, val: Self::Number) { - self.block_no = val; + fn blocks(&self) -> u32 { + self.blocks } - fn set_block_in_round(&mut self, val: Self::Number) { - self.block_in_round = val; + fn transactions(&self) -> u32 { + self.transactions } - fn set_round(&mut self, val: Self::Number) { - self.round = val; + fn set_block_number(&mut self, value: u32) { + self.block_number = value; } fn transfer_extrinsic( - &self, + &mut self, sender: &Self::AccountId, key: &Self::Secret, destination: &Self::AccountId, @@ -143,10 +110,12 @@ impl RuntimeAdapter for FactoryState { genesis_hash: &::Hash, prior_block_hash: &::Hash, ) -> ::Extrinsic { - let index = self.extract_index(&sender, prior_block_hash); - let phase = self.extract_phase(*prior_block_hash); + let phase = self.block_number() as Self::Phase; + let extra = Self::build_extra(self.index, phase); + self.index += 1; + sign::(CheckedExtrinsic { - signed: Some((sender.clone(), Self::build_extra(index, phase))), + signed: Some((sender.clone(), extra)), function: Call::Balances( BalancesCall::transfer( pallet_indices::address::Address::Id(destination.clone().into()), @@ -157,12 +126,12 @@ impl RuntimeAdapter for FactoryState { } fn inherent_extrinsics(&self) -> InherentData { - let timestamp = (self.block_no as u64 + 1) * MinimumPeriod::get(); + let timestamp = (self.block_number as u64 + 1) * MinimumPeriod::get(); let mut inherent = InherentData::new(); inherent.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) .expect("Failed putting timestamp inherent"); - inherent.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &self.block_no) + inherent.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &self.block_number) .expect("Failed putting finalized number inherent"); inherent } @@ -180,49 +149,16 @@ impl RuntimeAdapter for FactoryState { } /// Generates a random `AccountId` from `seed`. - fn gen_random_account_id(seed: &Self::Number) -> Self::AccountId { - let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(*seed)); + fn gen_random_account_id(seed: u32) -> Self::AccountId { + let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(seed)); AccountPublic::from(pair.public()).into_account() } /// Generates a random `Secret` from `seed`. - fn gen_random_account_secret(seed: &Self::Number) -> Self::Secret { - let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(*seed)); + fn gen_random_account_secret(seed: u32) -> Self::Secret { + let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(seed)); pair } - - fn extract_index( - &self, - _account_id: &Self::AccountId, - _block_hash: &::Hash, - ) -> Self::Index { - // TODO get correct index for account via api. See #2587. - // This currently prevents the factory from being used - // without a preceding purge of the database. - if self.mode == Mode::MasterToN || self.mode == Mode::MasterTo1 { - self.block_no() as Self::Index - } else { - match self.round() { - 0 => - // if round is 0 all transactions will be done with master as a sender - self.block_no() as Self::Index, - _ => - // if round is e.g. 1 every sender account will be new and not yet have - // any transactions done - 0 - } - } - } - - fn extract_phase( - &self, - _block_hash: ::Hash - ) -> Self::Phase { - // TODO get correct phase via api. See #2587. - // This currently prevents the factory from being used - // without a preceding purge of the database. - self.block_no() as Self::Phase - } } fn gen_seed_bytes(seed: u32) -> [u8; 32] { diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index f5b915a2bedd611904ecee96cf8f24f01140d881..789d6a6913fc9a4226fa56a81ec773d5c8d32f10 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -27,7 +27,6 @@ //! hasn't been tested. #![warn(missing_docs)] -#![warn(unused_extern_crates)] pub mod chain_spec; diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 2c500c6a1c1edd3946b1ea83d1316a5f18346c04..70dd0521dece317242a31e4d8bca6f6169063e16 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -30,7 +30,6 @@ use sc_service::{ AbstractService, ServiceBuilder, config::Configuration, error::{Error as ServiceError}, }; use sp_inherents::InherentDataProviders; -use sc_network::construct_simple_protocol; use sc_service::{Service, NetworkStatus}; use sc_client::{Client, LocalCallExecutor}; @@ -40,11 +39,6 @@ use node_executor::NativeExecutor; use sc_network::NetworkService; use sc_offchain::OffchainWorkers; -construct_simple_protocol! { - /// Demo protocol attachment for substrate. - pub struct NodeProtocol where Block = Block { } -} - /// Starts a `ServiceBuilder` for a full service. /// /// Use this macro if you don't actually need the full service, but just the builder in order to @@ -63,8 +57,7 @@ macro_rules! new_full_start { })? .with_transaction_pool(|config, client, _fetcher| { let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); - let pool = sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api)); - Ok(pool) + Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api))) })? .with_import_queue(|_config, client, mut select_chain, _transaction_pool| { let select_chain = select_chain.take() @@ -80,7 +73,6 @@ macro_rules! new_full_start { sc_consensus_babe::Config::get_or_compute(&*client)?, grandpa_block_import, client.clone(), - client.clone(), )?; let import_queue = sc_consensus_babe::import_queue( @@ -88,7 +80,6 @@ macro_rules! new_full_start { block_import.clone(), Some(Box::new(justification_import)), None, - client.clone(), client, inherent_data_providers.clone(), )?; @@ -96,8 +87,21 @@ macro_rules! new_full_start { import_setup = Some((block_import, grandpa_link, babe_link)); Ok(import_queue) })? - .with_rpc_extensions(|client, pool, _backend, fetcher, _remote_blockchain| -> Result { - Ok(node_rpc::create(client, pool, node_rpc::LightDeps::none(fetcher))) + .with_rpc_extensions(|builder| -> Result { + let babe_link = import_setup.as_ref().map(|s| &s.2) + .expect("BabeLink is present for full services or set up failed; qed."); + let deps = node_rpc::FullDeps { + client: builder.client().clone(), + pool: builder.pool(), + select_chain: builder.select_chain().cloned() + .expect("SelectChain is present for full services or set up failed; qed."), + babe: node_rpc::BabeDeps { + keystore: builder.keystore(), + babe_config: sc_consensus_babe::BabeLink::config(babe_link).clone(), + shared_epoch_changes: sc_consensus_babe::BabeLink::epoch_changes(babe_link).clone() + } + }; + Ok(node_rpc::create_full(deps)) })?; (builder, import_setup, inherent_data_providers) @@ -134,7 +138,7 @@ macro_rules! new_full { let (builder, mut import_setup, inherent_data_providers) = new_full_start!($config); - let service = builder.with_network_protocol(|_| Ok(crate::service::NodeProtocol::new()))? + let service = builder .with_finality_proof_provider(|client, backend| Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, client)) as _) )? @@ -203,48 +207,41 @@ macro_rules! new_full { gossip_duration: std::time::Duration::from_millis(333), justification_period: 512, name: Some(name), - observer_enabled: true, + observer_enabled: false, keystore, is_authority, }; - match (is_authority, disable_grandpa) { - (false, false) => { - // start the lightweight GRANDPA observer - service.spawn_task("grandpa-observer", grandpa::run_grandpa_observer( - config, - grandpa_link, - service.network(), - service.on_exit(), - service.spawn_task_handle(), - )?); - }, - (true, false) => { - // start the full GRANDPA voter - let grandpa_config = grandpa::GrandpaParams { - config: config, - link: grandpa_link, - network: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - on_exit: service.on_exit(), - telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), - executor: service.spawn_task_handle(), - }; - // the GRANDPA voter task is considered infallible, i.e. - // if it fails we take down the service with it. - service.spawn_essential_task( - "grandpa-voter", - grandpa::run_grandpa_voter(grandpa_config)? - ); - }, - (_, true) => { - grandpa::setup_disabled_grandpa( - service.client(), - &inherent_data_providers, - service.network(), - )?; - }, + let enable_grandpa = !disable_grandpa; + if enable_grandpa { + // start the full GRANDPA voter + // NOTE: non-authorities could run the GRANDPA observer protocol, but at + // this point the full voter should provide better guarantees of block + // and vote data availability than the observer. The observer has not + // been tested extensively yet and having most nodes in a network run it + // could lead to finality stalls. + let grandpa_config = grandpa::GrandpaParams { + config, + link: grandpa_link, + network: service.network(), + inherent_data_providers: inherent_data_providers.clone(), + on_exit: service.on_exit(), + telemetry_on_connect: Some(service.telemetry_on_connect_stream()), + voting_rule: grandpa::VotingRulesBuilder::default().build(), + }; + + // the GRANDPA voter task is considered infallible, i.e. + // if it fails we take down the service with it. + service.spawn_essential_task( + "grandpa-voter", + grandpa::run_grandpa_voter(grandpa_config)? + ); + } else { + grandpa::setup_disabled_grandpa( + service.client(), + &inherent_data_providers, + service.network(), + )?; } Ok((service, inherent_data_providers)) @@ -254,9 +251,7 @@ macro_rules! new_full { }} } -#[allow(dead_code)] type ConcreteBlock = node_primitives::Block; -#[allow(dead_code)] type ConcreteClient = Client< Backend, @@ -265,9 +260,7 @@ type ConcreteClient = ConcreteBlock, node_runtime::RuntimeApi >; -#[allow(dead_code)] type ConcreteBackend = Backend; -#[allow(dead_code)] type ConcreteTransactionPool = sc_transaction_pool::BasicPool< sc_transaction_pool::FullChainApi, ConcreteBlock @@ -284,7 +277,7 @@ pub fn new_full(config: NodeConfiguration) ConcreteClient, LongestChain, NetworkStatus, - NetworkService::Hash>, + NetworkService::Hash>, ConcreteTransactionPool, OffchainWorkers< ConcreteClient, @@ -336,7 +329,6 @@ pub fn new_light(config: NodeConfiguration) sc_consensus_babe::Config::get_or_compute(&*client)?, grandpa_block_import, client.clone(), - client.clone(), )?; let import_queue = sc_consensus_babe::import_queue( @@ -345,24 +337,29 @@ pub fn new_light(config: NodeConfiguration) None, Some(Box::new(finality_proof_import)), client.clone(), - client, inherent_data_providers.clone(), )?; Ok((import_queue, finality_proof_request_builder)) })? - .with_network_protocol(|_| Ok(NodeProtocol::new()))? .with_finality_proof_provider(|client, backend| Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) )? - .with_rpc_extensions(|client, pool, _backend, fetcher, remote_blockchain| -> Result { - let fetcher = fetcher + .with_rpc_extensions(|builder,| -> + Result + { + let fetcher = builder.fetcher() .ok_or_else(|| "Trying to start node RPC without active fetcher")?; - let remote_blockchain = remote_blockchain + let remote_blockchain = builder.remote_backend() .ok_or_else(|| "Trying to start node RPC without active remote blockchain")?; - let light_deps = node_rpc::LightDeps { remote_blockchain, fetcher }; - Ok(node_rpc::create(client, pool, Some(light_deps))) + let light_deps = node_rpc::LightDeps { + remote_blockchain, + fetcher, + client: builder.client().clone(), + pool: builder.pool(), + }; + Ok(node_rpc::create_light(light_deps)) })? .build()?; @@ -371,7 +368,7 @@ pub fn new_light(config: NodeConfiguration) #[cfg(test)] mod tests { - use std::{sync::Arc, collections::HashMap, borrow::Cow, any::Any}; + use std::{sync::Arc, borrow::Cow, any::Any}; use sc_consensus_babe::{ CompatibleDigestItem, BabeIntermediate, INTERMEDIATE_KEY }; @@ -487,7 +484,7 @@ mod tests { |config| { let mut setup_handles = None; new_full!(config, | - block_import: &sc_consensus_babe::BabeBlockImport<_, _, Block, _, _, _>, + block_import: &sc_consensus_babe::BabeBlockImport, babe_link: &sc_consensus_babe::BabeLink, | { setup_handles = Some((block_import.clone(), babe_link.clone())); @@ -559,27 +556,14 @@ mod tests { ); slot_num += 1; - let params = BlockImportParams { - origin: BlockOrigin::File, - header: new_header, - justification: None, - post_digests: vec![item], - body: Some(new_body), - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: { - let mut intermediates = HashMap::new(); - intermediates.insert( - Cow::from(INTERMEDIATE_KEY), - Box::new(BabeIntermediate { epoch }) as Box, - ); - intermediates - }, - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; + let mut params = BlockImportParams::new(BlockOrigin::File, new_header); + params.post_digests.push(item); + params.body = Some(new_body); + params.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(BabeIntermediate { epoch }) as Box, + ); + params.fork_choice = Some(ForkChoiceStrategy::LongestChain); block_import.import_block(params, Default::default()) .expect("error importing test block"); diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs new file mode 100644 index 0000000000000000000000000000000000000000..2eca71a5b5978de1e1de20873b6584a7d5c703b0 --- /dev/null +++ b/bin/node/cli/tests/build_spec_works.rs @@ -0,0 +1,37 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +#[test] +fn build_spec_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + let output = Command::new(cargo_bin("substrate")) + .args(&["build-spec", "--dev", "-d"]) + .arg(base_path.path()) + .output() + .unwrap(); + assert!(output.status.success()); + + // Make sure that the `dev` chain folder exists, but the `db` doesn't + assert!(base_path.path().join("chains/dev/").exists()); + assert!(!base_path.path().join("chains/dev/db").exists()); + + let _value: serde_json::Value = serde_json::from_slice(output.stdout.as_slice()).unwrap(); +} diff --git a/bin/node/transaction-factory/src/modes.rs b/bin/node/cli/tests/check_block_works.rs similarity index 52% rename from bin/node/transaction-factory/src/modes.rs rename to bin/node/cli/tests/check_block_works.rs index 5deab7635e114c16469fd3babbda2d1004b20641..6bfb82a8bfafb84c1b041b87dc093e2481193ec5 100644 --- a/bin/node/transaction-factory/src/modes.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,27 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! The transaction factory can operate in different modes. See -//! the `simple_mode` and `complex_mode` modules for details. +#![cfg(unix)] -use std::str::FromStr; +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; -/// Token distribution modes. -#[derive(Debug, Clone, PartialEq)] -pub enum Mode { - MasterToN, - MasterTo1, - MasterToNToM -} +mod common; + +#[test] +fn check_block_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + common::run_dev_node_for_a_while(base_path.path()); -impl FromStr for Mode { - type Err = String; - fn from_str(mode: &str) -> Result { - match mode { - "MasterToN" => Ok(Mode::MasterToN), - "MasterTo1" => Ok(Mode::MasterTo1), - "MasterToNToM" => Ok(Mode::MasterToNToM), - _ => Err(format!("Invalid mode: {}", mode)), - } - } + let status = Command::new(cargo_bin("substrate")) + .args(&["check-block", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg("1") + .status() + .unwrap(); + assert!(status.success()); } diff --git a/bin/node/cli/tests/common.rs b/bin/node/cli/tests/common.rs index 96060bf85d9f1917917a6c498918b7b5c822c0a4..34e371195c16b20e216a9426f238ce6b78bcd257 100644 --- a/bin/node/cli/tests/common.rs +++ b/bin/node/cli/tests/common.rs @@ -14,21 +14,53 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{process::{Child, ExitStatus}, thread, time::Duration}; +#![cfg(unix)] +#![allow(dead_code)] -/// Wait for the given `child` the given ammount of `secs`. +use std::{process::{Child, ExitStatus}, thread, time::Duration, path::Path}; +use assert_cmd::cargo::cargo_bin; +use std::{convert::TryInto, process::Command}; +use nix::sys::signal::{kill, Signal::SIGINT}; +use nix::unistd::Pid; + +/// Wait for the given `child` the given number of `secs`. /// /// Returns the `Some(exit status)` or `None` if the process did not finish in the given time. pub fn wait_for(child: &mut Child, secs: usize) -> Option { - for _ in 0..secs { + for i in 0..secs { match child.try_wait().unwrap() { - Some(status) => return Some(status), + Some(status) => { + if i > 5 { + eprintln!("Child process took {} seconds to exit gracefully", i); + } + return Some(status) + }, None => thread::sleep(Duration::from_secs(1)), } } - eprintln!("Took to long to exit. Killing..."); + eprintln!("Took too long to exit (> {} seconds). Killing...", secs); let _ = child.kill(); child.wait().unwrap(); None } + +/// Run the node for a while (30 seconds) +pub fn run_dev_node_for_a_while(base_path: &Path) { + let mut cmd = Command::new(cargo_bin("substrate")); + + let mut cmd = cmd + .args(&["--dev"]) + .arg("-d") + .arg(base_path) + .spawn() + .unwrap(); + + // Let it produce some blocks. + thread::sleep(Duration::from_secs(30)); + assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); + + // Stop the process + kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); + assert!(wait_for(&mut cmd, 40).map(|x| x.success()).unwrap_or_default()); +} diff --git a/utils/grafana-data-source/test/src/main.rs b/bin/node/cli/tests/factory.rs similarity index 51% rename from utils/grafana-data-source/test/src/main.rs rename to bin/node/cli/tests/factory.rs index 53deaffc3beb6088acc71bfee202bfbae41f0ef4..2930cd52e2e16e497d197849a9833d492c3868d7 100644 --- a/utils/grafana-data-source/test/src/main.rs +++ b/bin/node/cli/tests/factory.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,31 +14,27 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use grafana_data_source::{run_server, record_metrics}; -use std::time::Duration; -use rand::Rng; -use futures::{future::join, executor}; +#![cfg(unix)] -async fn randomness() { - loop { - futures_timer::Delay::new(Duration::from_secs(1)).await; +use assert_cmd::cargo::cargo_bin; +use std::process::{Command, Stdio}; +use tempfile::tempdir; - let random = rand::thread_rng().gen_range(0.0, 1000.0); +mod common; - let result = record_metrics!( - "random data" => random, - "random^2" => random * random, - ); +#[test] +fn factory_works() { + let base_path = tempdir().expect("could not create a temp dir"); - if let Err(error) = result { - eprintln!("{}", error); - } - } -} + let status = Command::new(cargo_bin("substrate")) + .stdout(Stdio::null()) + .args(&["factory", "--dev", "-d"]) + .arg(base_path.path()) + .status() + .unwrap(); + assert!(status.success()); -fn main() { - executor::block_on(join( - run_server("127.0.0.1:9955".parse().unwrap()), - randomness() - )).0.unwrap(); + // Make sure that the `dev` chain folder exists & `db` + assert!(base_path.path().join("chains/dev/").exists()); + assert!(base_path.path().join("chains/dev/db").exists()); } diff --git a/bin/node/cli/tests/import_export_and_revert_work.rs b/bin/node/cli/tests/import_export_and_revert_work.rs new file mode 100644 index 0000000000000000000000000000000000000000..131265e3b4ab9bd868b9728bca6cda558d2982f2 --- /dev/null +++ b/bin/node/cli/tests/import_export_and_revert_work.rs @@ -0,0 +1,59 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::{process::Command, fs}; +use tempfile::tempdir; + +mod common; + +#[test] +fn import_export_and_revert_work() { + let base_path = tempdir().expect("could not create a temp dir"); + let exported_blocks = base_path.path().join("exported_blocks"); + + common::run_dev_node_for_a_while(base_path.path()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["export-blocks", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg(&exported_blocks) + .status() + .unwrap(); + assert!(status.success()); + + let metadata = fs::metadata(&exported_blocks).unwrap(); + assert!(metadata.len() > 0, "file exported_blocks should not be empty"); + + let _ = fs::remove_dir_all(base_path.path().join("db")); + + let status = Command::new(cargo_bin("substrate")) + .args(&["import-blocks", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg(&exported_blocks) + .status() + .unwrap(); + assert!(status.success()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["revert", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .status() + .unwrap(); + assert!(status.success()); +} diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs new file mode 100644 index 0000000000000000000000000000000000000000..441b08ccf46da1b1ef70ec49a7a46015d94f3c4c --- /dev/null +++ b/bin/node/cli/tests/inspect_works.rs @@ -0,0 +1,38 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +mod common; + +#[test] +fn inspect_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + common::run_dev_node_for_a_while(base_path.path()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["inspect", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .args(&["block", "1"]) + .status() + .unwrap(); + assert!(status.success()); +} diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index e6b71db9910487a4e55f2fdf117e48c833b120bb..020259d0c595a57a01b8a9b113522d826e1de912 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -15,39 +15,27 @@ // along with Substrate. If not, see . use assert_cmd::cargo::cargo_bin; -use std::{convert::TryInto, process::Command, thread, time::Duration, fs, path::PathBuf}; +use std::process::Command; +use tempfile::tempdir; mod common; #[test] #[cfg(unix)] fn purge_chain_works() { - use nix::sys::signal::{kill, Signal::SIGINT}; - use nix::unistd::Pid; + let base_path = tempdir().expect("could not create a temp dir"); - let base_path = "purge_chain_test"; - - let _ = fs::remove_dir_all(base_path); - let mut cmd = Command::new(cargo_bin("substrate")) - .args(&["--dev", "-d", base_path]) - .spawn() - .unwrap(); - - // Let it produce some blocks. - thread::sleep(Duration::from_secs(30)); - assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); - - // Stop the process - kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); - assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default()); + common::run_dev_node_for_a_while(base_path.path()); let status = Command::new(cargo_bin("substrate")) - .args(&["purge-chain", "--dev", "-d", base_path, "-y"]) + .args(&["purge-chain", "--dev", "-d"]) + .arg(base_path.path()) + .arg("-y") .status() .unwrap(); assert!(status.success()); // Make sure that the `dev` chain folder exists, but the `db` is deleted. - assert!(PathBuf::from(base_path).join("chains/dev/").exists()); - assert!(!PathBuf::from(base_path).join("chains/dev/db").exists()); + assert!(base_path.path().join("chains/dev/").exists()); + assert!(!base_path.path().join("chains/dev/db").exists()); } diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index 6ab719de9196d89afb60c9f8affe26ceb685ffc7..67efedccbe771a65e892b5dbe94bac702d3c01ce 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -15,7 +15,8 @@ // along with Substrate. If not, see . use assert_cmd::cargo::cargo_bin; -use std::{convert::TryInto, process::Command, thread, time::Duration, fs}; +use std::{convert::TryInto, process::Command, thread, time::Duration}; +use tempfile::tempdir; mod common; @@ -26,19 +27,20 @@ fn running_the_node_works_and_can_be_interrupted() { use nix::unistd::Pid; fn run_command_and_kill(signal: Signal) { - let _ = fs::remove_dir_all("interrupt_test"); + let base_path = tempdir().expect("could not create a temp dir"); let mut cmd = Command::new(cargo_bin("substrate")) - .args(&["--dev", "-d", "interrupt_test"]) + .args(&["--dev", "-d"]) + .arg(base_path.path()) .spawn() .unwrap(); - thread::sleep(Duration::from_secs(30)); + thread::sleep(Duration::from_secs(20)); assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); kill(Pid::from_raw(cmd.id().try_into().unwrap()), signal).unwrap(); assert_eq!( common::wait_for(&mut cmd, 30).map(|x| x.success()), Some(true), - "the pocess must exit gracefully after signal {}", + "the process must exit gracefully after signal {}", signal, ); } diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 0d68b705228b5df4b91a4dccd68f482e99d9ac09..a4eacbb822b6a9e9925cffb52b8d25f8ac6348e9 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -1,39 +1,42 @@ [package] name = "node-executor" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sc-executor = { version = "0.8", path = "../../../client/executor" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-state-machine = { version = "0.8", path = "../../../primitives/state-machine" } -sp-trie = { version = "2.0.0", path = "../../../primitives/trie" } -trie-root = "0.15.2" +node-primitives = { version = "2.0.0-alpha.1", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.1", path = "../runtime" } +sc-executor = { version = "0.8.0-alpha.1", path = "../../../client/executor" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../../primitives/state-machine" } +sp-trie = { version = "2.0.0-alpha.1", path = "../../../primitives/trie" } +trie-root = "0.16.0" +frame-benchmarking = { version = "2.0.0-alpha.1", path = "../../../frame/benchmarking" } [dev-dependencies] criterion = "0.3.0" -frame-support = { version = "2.0.0", path = "../../../frame/support" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -node-testing = { version = "2.0.0", path = "../testing" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0", path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0", path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -pallet-session = { version = "2.0.0", path = "../../../frame/session" } -pallet-timestamp = { version = "2.0.0", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0", path = "../../../frame/treasury" } -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } +frame-support = { version = "2.0.0-alpha.1", path = "../../../frame/support" } +frame-system = { version = "2.0.0-alpha.1", path = "../../../frame/system" } +node-testing = { version = "2.0.0-alpha.1", path = "../testing" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../../../frame/balances" } +pallet-contracts = { version = "2.0.0-alpha.1", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-alpha.1", path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-alpha.1", path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-alpha.1", path = "../../../frame/indices" } +pallet-session = { version = "2.0.0-alpha.1", path = "../../../frame/session" } +pallet-timestamp = { version = "2.0.0-alpha.1", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-alpha.1", path = "../../../frame/treasury" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../../primitives/application-crypto" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } wabt = "0.9.2" [features] diff --git a/bin/node/executor/src/lib.rs b/bin/node/executor/src/lib.rs index 812c018502f1b606cbd3db366420ae64f34ab6e4..bcc7f48507398cd0ea15319460f8d13e6058f2ec 100644 --- a/bin/node/executor/src/lib.rs +++ b/bin/node/executor/src/lib.rs @@ -25,5 +25,6 @@ use sc_executor::native_executor_instance; native_executor_instance!( pub Executor, node_runtime::api::dispatch, - node_runtime::native_version + node_runtime::native_version, + frame_benchmarking::benchmarking::HostFunctions, ); diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index b306852f4547d04a49e296bbf5b2e17ff48969af..eb629029c6dd1391a24ae7f9c5893bd48c552f0d 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -35,7 +35,7 @@ use frame_system::{self, EventRecord, Phase}; use node_runtime::{ Header, Block, UncheckedExtrinsic, CheckedExtrinsic, Call, Runtime, Balances, - System, TransactionPayment, Event, TransactionBaseFee, TransactionByteFee, CreationFee, + System, TransactionPayment, Event, TransactionBaseFee, TransactionByteFee, constants::currency::*, }; use node_primitives::{Balance, Hash}; @@ -163,15 +163,12 @@ fn block_with_size(time: u64, nonce: u32, size: usize) -> (Vec, Hash) { fn panic_execution_with_foreign_code_gives_error() { let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (69u128, 0u128, 0u128, 0u128).encode() + >::hashed_key_for(alice()) => { + (69u128, 0u8, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { 69_u128.encode() }, - >::hashed_key().to_vec() => { - 0_u128.encode() - }, >::hashed_key_for(0) => { vec![0u8; 32] } @@ -202,15 +199,12 @@ fn panic_execution_with_foreign_code_gives_error() { fn bad_extrinsic_with_native_equivalent_code_gives_error() { let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (69u128, 0u128, 0u128, 0u128).encode() + >::hashed_key_for(alice()) => { + (0u32, 0u8, 69u128, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { 69_u128.encode() }, - >::hashed_key().to_vec() => { - 0_u128.encode() - }, >::hashed_key_for(0) => { vec![0u8; 32] } @@ -241,13 +235,12 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() { fn successful_execution_with_native_equivalent_code_gives_ok() { let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS, 0u128, 0u128, 0u128).encode() + >::hashed_key_for(alice()) => { + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() }, - >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] ], children: map![], @@ -274,7 +267,7 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { assert!(r.is_ok()); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm) + CreationFee::get(); + let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); @@ -284,13 +277,12 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { fn successful_execution_with_foreign_code_gives_ok() { let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS, 0u128, 0u128, 0u128).encode() + >::hashed_key_for(alice()) => { + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() }, - >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] ], children: map![], @@ -317,7 +309,7 @@ fn successful_execution_with_foreign_code_gives_ok() { assert!(r.is_ok()); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm) + CreationFee::get(); + let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); @@ -348,14 +340,14 @@ fn full_native_block_import_works() { let events = vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: Event::system(frame_system::Event::ExtrinsicSuccess( + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( DispatchInfo { weight: 10000, class: DispatchClass::Operational, pays_fee: true } )), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984800000000)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -364,13 +356,12 @@ fn full_native_block_import_works() { alice().into(), bob().into(), 69 * DOLLARS, - 0, )), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::system(frame_system::Event::ExtrinsicSuccess( + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } )), topics: vec![], @@ -390,25 +381,26 @@ fn full_native_block_import_works() { ).0.unwrap(); t.execute_with(|| { + let fees = transfer_fee(&xt(), fm); assert_eq!( Balances::total_balance(&alice()), - alice_last_known_balance - 10 * DOLLARS - transfer_fee(&xt(), fm), + alice_last_known_balance - 10 * DOLLARS - fees, ); assert_eq!( Balances::total_balance(&bob()), - 179 * DOLLARS - transfer_fee(&xt(), fm), + 179 * DOLLARS - fees, ); let events = vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: Event::system(frame_system::Event::ExtrinsicSuccess( + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( DispatchInfo { weight: 10000, class: DispatchClass::Operational, pays_fee: true } )), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984788199392)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -418,21 +410,20 @@ fn full_native_block_import_works() { bob().into(), alice().into(), 5 * DOLLARS, - 0, ) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::system(frame_system::Event::ExtrinsicSuccess( + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } )), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984788199392)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -442,14 +433,13 @@ fn full_native_block_import_works() { alice().into(), bob().into(), 15 * DOLLARS, - 0, ) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::system(frame_system::Event::ExtrinsicSuccess( + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } )), topics: vec![], @@ -712,13 +702,9 @@ fn native_big_block_import_fails_on_fallback() { fn panic_execution_gives_error() { let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (0_u128, 0_u128, 0_u128, 0_u128).encode() - }, >::hashed_key().to_vec() => { 0_u128.encode() }, - >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] ], children: map![], @@ -747,13 +733,12 @@ fn panic_execution_gives_error() { fn successful_execution_gives_ok() { let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (111 * DOLLARS, 0u128, 0u128, 0u128).encode() + >::hashed_key_for(alice()) => { + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() }, - >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] ], children: map![], @@ -777,11 +762,11 @@ fn successful_execution_gives_ok() { ).0.unwrap().into_encoded(); ApplyExtrinsicResult::decode(&mut &r[..]) .unwrap() - .expect("Extrinsic could be applied") - .expect("Extrinsic did not fail"); + .expect("Extrinsic could not be applied") + .expect("Extrinsic failed"); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm) + CreationFee::get(); + let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); diff --git a/bin/node/executor/tests/fees.rs b/bin/node/executor/tests/fees.rs index 155cefa4ccada20b0b349221e6c56d6328ac1d8f..024c02bd6aaf915be6068546842a7445abe78562 100644 --- a/bin/node/executor/tests/fees.rs +++ b/bin/node/executor/tests/fees.rs @@ -25,13 +25,13 @@ use sp_core::{ storage::Storage, }; use sp_runtime::{ - Fixed64, + Fixed64, Perbill, traits::Convert, }; use node_runtime::{ - CheckedExtrinsic, Call, Runtime, Balances, - TransactionPayment, TransactionBaseFee, TransactionByteFee, - WeightFeeCoefficient, constants::currency::*, + CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, TransactionBaseFee, + TransactionByteFee, WeightFeeCoefficient, + constants::currency::*, }; use node_runtime::impls::LinearWeightToFee; use node_primitives::Balance; @@ -60,12 +60,12 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { GENESIS_HASH.into(), vec![ CheckedExtrinsic { - signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(42 * 1000)), + signed: None, + function: Call::Timestamp(pallet_timestamp::Call::set(42 * 1000)), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(0, 0))), - function: Call::System(frame_system::Call::fill_block()), + function: Call::System(frame_system::Call::fill_block(Perbill::from_percent(90))), } ] ); @@ -77,8 +77,8 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { block1.1.clone(), vec![ CheckedExtrinsic { - signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(52 * 1000)), + signed: None, + function: Call::Timestamp(pallet_timestamp::Call::set(52 * 1000)), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(1, 0))), @@ -87,7 +87,11 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { ] ); - println!("++ Block 1 size: {} / Block 2 size {}", block1.0.encode().len(), block2.0.encode().len()); + println!( + "++ Block 1 size: {} / Block 2 size {}", + block1.0.encode().len(), + block2.0.encode().len(), + ); // execute a big block. executor_call:: _>( @@ -134,16 +138,15 @@ fn transaction_fee_is_correct_ultimate() { // (this baed on assigning 0.1 CENT to the cheapest tx with `weight = 100`) let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ - >::hashed_key_for(alice()) => { - (100 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() + >::hashed_key_for(alice()) => { + (0u32, 0u8, 100 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() }, - >::hashed_key_for(bob()) => { - (10 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() + >::hashed_key_for(bob()) => { + (0u32, 0u8, 10 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() }, >::hashed_key().to_vec() => { - (110 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() + (110 * DOLLARS).encode() }, - >::hashed_key().to_vec() => vec![0u8; 16], >::hashed_key_for(0) => vec![0u8; 32] ], children: map![], diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 40c0c6b80fefd2143231ac7adf0de498a3ac2b9c..1a92aeca6ba77b61597f3f0d19317f67adca2b03 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -167,8 +167,9 @@ fn submitted_transaction_should_be_valid() { // add balance to the account let author = extrinsic.signature.clone().unwrap().0; let address = Indices::lookup(author).unwrap(); - let account = pallet_balances::AccountData { free: 5_000_000_000_000, ..Default::default() }; - >::insert(&address, account); + let data = pallet_balances::AccountData { free: 5_000_000_000_000, ..Default::default() }; + let account = frame_system::AccountInfo { nonce: 0u32, refcount: 0u8, data }; + >::insert(&address, account); // check validity let res = Executive::validate_transaction(extrinsic); diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..353daf7a4f2d6fcf0f284ec5de9c27ffc53f25ad --- /dev/null +++ b/bin/node/inspect/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "node-inspect" +version = "0.8.0-alpha.1" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.0.0" } +derive_more = "0.99" +log = "0.4.8" +sc-cli = { version = "0.8.0-alpha.1", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../../client/api" } +sc-service = { version = "0.8.0-alpha.1", default-features = false, path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +structopt = "0.3.8" diff --git a/bin/node/inspect/src/cli.rs b/bin/node/inspect/src/cli.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d51bd5848f17d308933fdcb69d429504fa8445e --- /dev/null +++ b/bin/node/inspect/src/cli.rs @@ -0,0 +1,62 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Structs to easily compose inspect sub-command for CLI. + +use std::fmt::Debug; +use sc_cli::{ImportParams, SharedParams}; +use structopt::StructOpt; + +/// The `inspect` command used to print decoded chain data. +#[derive(Debug, StructOpt, Clone)] +pub struct InspectCmd { + #[allow(missing_docs)] + #[structopt(flatten)] + pub command: InspectSubCmd, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, +} + +/// A possible inspect sub-commands. +#[derive(Debug, StructOpt, Clone)] +pub enum InspectSubCmd { + /// Decode block with native version of runtime and print out the details. + Block { + /// Address of the block to print out. + /// + /// Can be either a block hash (no 0x prefix) or a number to retrieve existing block, + /// or a 0x-prefixed bytes hex string, representing SCALE encoding of + /// a block. + #[structopt(value_name = "HASH or NUMBER or BYTES")] + input: String, + }, + /// Decode extrinsic with native version of runtime and print out the details. + Extrinsic { + /// Address of an extrinsic to print out. + /// + /// Can be either a block hash (no 0x prefix) or number and the index, in the form + /// of `{block}:{index}` or a 0x-prefixed bytes hex string, + /// representing SCALE encoding of an extrinsic. + #[structopt(value_name = "BLOCK:INDEX or BYTES")] + input: String, + }, +} diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs new file mode 100644 index 0000000000000000000000000000000000000000..71e70e3e44fd4cda6ac398a385d1785c765079b9 --- /dev/null +++ b/bin/node/inspect/src/command.rs @@ -0,0 +1,204 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Command ran by the CLI + +use std::{ + fmt::Debug, + str::FromStr, +}; + +use crate::cli::{InspectCmd, InspectSubCmd}; +use crate::{Inspector, PrettyPrinter}; + +impl InspectCmd { + /// Initialize + pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { + self.shared_params.init(version) + } + + /// Parse CLI arguments and initialize given config. + pub fn update_config( + &self, + mut config: &mut sc_service::config::Configuration, + spec_factory: impl FnOnce(&str) -> Result>, String>, + version: &sc_cli::VersionInfo, + ) -> sc_cli::Result<()> where + G: sc_service::RuntimeGenesis, + E: sc_service::ChainSpecExtension, + { + self.shared_params.update_config(config, spec_factory, version)?; + + // make sure to configure keystore + config.use_in_memory_keystore()?; + + // and all import params (especially pruning that has to match db meta) + self.import_params.update_config( + &mut config, + sc_service::Roles::FULL, + self.shared_params.dev, + )?; + + Ok(()) + } + + /// Run the inspect command, passing the inspector. + pub fn run( + self, + inspect: Inspector, + ) -> sc_cli::Result<()> where + B: sp_runtime::traits::Block, + B::Hash: FromStr, + P: PrettyPrinter, + { + match self.command { + InspectSubCmd::Block { input } => { + let input = input.parse()?; + let res = inspect.block(input) + .map_err(|e| format!("{}", e))?; + println!("{}", res); + Ok(()) + }, + InspectSubCmd::Extrinsic { input } => { + let input = input.parse()?; + let res = inspect.extrinsic(input) + .map_err(|e| format!("{}", e))?; + println!("{}", res); + Ok(()) + }, + } + } +} + +/// A block to retrieve. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockAddress { + /// Get block by hash. + Hash(Hash), + /// Get block by number. + Number(Number), + /// Raw SCALE-encoded bytes. + Bytes(Vec), +} + +impl FromStr for BlockAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // try to parse hash first + if let Ok(hash) = s.parse() { + return Ok(Self::Hash(hash)) + } + + // then number + if let Ok(number) = s.parse() { + return Ok(Self::Number(number)) + } + + // then assume it's bytes (hex-encoded) + sp_core::bytes::from_hex(s) + .map(Self::Bytes) + .map_err(|e| format!( + "Given string does not look like hash or number. It could not be parsed as bytes either: {}", + e + )) + } +} + +/// An extrinsic address to decode and print out. +#[derive(Debug, Clone, PartialEq)] +pub enum ExtrinsicAddress { + /// Extrinsic as part of existing block. + Block(BlockAddress, usize), + /// Raw SCALE-encoded extrinsic bytes. + Bytes(Vec), +} + +impl FromStr for ExtrinsicAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // first try raw bytes + if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { + return Ok(bytes) + } + + // split by a bunch of different characters + let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); + let block = it.next() + .expect("First element of split iterator is never empty; qed") + .parse()?; + + let index = it.next() + .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? + .parse() + .map_err(|e| format!("Invalid index format: {}", e))?; + + Ok(Self::Block(block, index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::hash::H160 as Hash; + + #[test] + fn should_parse_block_strings() { + type BlockAddress = super::BlockAddress; + + let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); + let b1 = BlockAddress::from_str("1234"); + let b2 = BlockAddress::from_str("0"); + let b3 = BlockAddress::from_str("0x0012345f"); + + + assert_eq!(b0, Ok(BlockAddress::Hash( + "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() + ))); + assert_eq!(b1, Ok(BlockAddress::Number(1234))); + assert_eq!(b2, Ok(BlockAddress::Number(0))); + assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } + + #[test] + fn should_parse_extrinsic_address() { + type BlockAddress = super::BlockAddress; + type ExtrinsicAddress = super::ExtrinsicAddress; + + let e0 = ExtrinsicAddress::from_str("1234"); + let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); + let b1 = ExtrinsicAddress::from_str("1234:0"); + let b2 = ExtrinsicAddress::from_str("0 0"); + let b3 = ExtrinsicAddress::from_str("0x0012345f"); + + + assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); + assert_eq!(b0, Ok(ExtrinsicAddress::Block( + BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), + 5 + ))); + assert_eq!(b1, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(1234), + 0 + ))); + assert_eq!(b2, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(0), + 0 + ))); + assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } +} diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..cd32f08e9fe59309bc3f4daf33035a763c1b242b --- /dev/null +++ b/bin/node/inspect/src/lib.rs @@ -0,0 +1,207 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! A CLI extension for substrate node, adding sub-command to pretty print debug info +//! about blocks and extrinsics. +//! +//! The blocks and extrinsics can either be retrieved from the database (on-chain), +//! or a raw SCALE-encoding can be provided. + +#![warn(missing_docs)] + +pub mod cli; +pub mod command; + +use std::{ + fmt, + marker::PhantomData +}; +use codec::{Encode, Decode}; +use sc_client_api::BlockBody; +use sp_blockchain::HeaderBackend; +use sp_core::hexdisplay::HexDisplay; +use sp_runtime::{ + generic::BlockId, + traits::{Block, HashFor, NumberFor, Hash} +}; + +use command::{BlockAddress, ExtrinsicAddress}; + +/// A helper type for a generic block input. +pub type BlockAddressFor = BlockAddress< + as Hash>::Output, + NumberFor +>; + +/// A Pretty formatter implementation. +pub trait PrettyPrinter { + /// Nicely format block. + fn fmt_block(&self, fmt: &mut fmt::Formatter, block: &TBlock) -> fmt::Result; + /// Nicely format extrinsic. + fn fmt_extrinsic(&self, fmt: &mut fmt::Formatter, extrinsic: &TBlock::Extrinsic) -> fmt::Result; +} + +/// Default dummy debug printer. +#[derive(Default)] +pub struct DebugPrinter; +impl PrettyPrinter for DebugPrinter { + fn fmt_block(&self, fmt: &mut fmt::Formatter, block: &TBlock) -> fmt::Result { + writeln!(fmt, "Header:")?; + writeln!(fmt, "{:?}", block.header())?; + writeln!(fmt, "Block bytes: {:?}", HexDisplay::from(&block.encode()))?; + writeln!(fmt, "Extrinsics ({})", block.extrinsics().len())?; + for (idx, ex) in block.extrinsics().iter().enumerate() { + writeln!(fmt, "- {}:", idx)?; + >::fmt_extrinsic(self, fmt, ex)?; + } + Ok(()) + } + + fn fmt_extrinsic(&self, fmt: &mut fmt::Formatter, extrinsic: &TBlock::Extrinsic) -> fmt::Result { + writeln!(fmt, " {:?}", extrinsic)?; + writeln!(fmt, " Bytes: {:?}", HexDisplay::from(&extrinsic.encode()))?; + Ok(()) + } +} + +/// Aggregated error for `Inspector` operations. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + /// Could not decode Block or Extrinsic. + Codec(codec::Error), + /// Error accessing blockchain DB. + Blockchain(sp_blockchain::Error), + /// Given block has not been found. + NotFound(String), +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match *self { + Self::Codec(ref e) => Some(e), + Self::Blockchain(ref e) => Some(e), + Self::NotFound(_) => None, + } + } +} + +/// A helper trait to access block headers and bodies. +pub trait ChainAccess: + HeaderBackend + + BlockBody +{} + +impl ChainAccess for T where + TBlock: Block, + T: sp_blockchain::HeaderBackend + sc_client_api::BlockBody, +{} + +/// Blockchain inspector. +pub struct Inspector = DebugPrinter> { + printer: TPrinter, + chain: Box>, + _block: PhantomData, +} + +impl> Inspector { + /// Create new instance of the inspector with default printer. + pub fn new( + chain: impl ChainAccess + 'static, + ) -> Self where TPrinter: Default { + Self::with_printer(chain, Default::default()) + } + + /// Customize pretty-printing of the data. + pub fn with_printer( + chain: impl ChainAccess + 'static, + printer: TPrinter, + ) -> Self { + Inspector { + chain: Box::new(chain) as _, + printer, + _block: Default::default(), + } + } + + /// Get a pretty-printed block. + pub fn block(&self, input: BlockAddressFor) -> Result { + struct BlockPrinter<'a, A, B>(A, &'a B); + impl<'a, A: Block, B: PrettyPrinter> fmt::Display for BlockPrinter<'a, A, B> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.1.fmt_block(fmt, &self.0) + } + } + + let block = self.get_block(input)?; + Ok(format!("{}", BlockPrinter(block, &self.printer))) + } + + fn get_block(&self, input: BlockAddressFor) -> Result { + Ok(match input { + BlockAddress::Bytes(bytes) => { + TBlock::decode(&mut &*bytes)? + }, + BlockAddress::Number(number) => { + let id = BlockId::number(number); + let not_found = format!("Could not find block {:?}", id); + let body = self.chain.block_body(&id)? + .ok_or_else(|| Error::NotFound(not_found.clone()))?; + let header = self.chain.header(id)? + .ok_or_else(|| Error::NotFound(not_found.clone()))?; + TBlock::new(header, body) + }, + BlockAddress::Hash(hash) => { + let id = BlockId::hash(hash); + let not_found = format!("Could not find block {:?}", id); + let body = self.chain.block_body(&id)? + .ok_or_else(|| Error::NotFound(not_found.clone()))?; + let header = self.chain.header(id)? + .ok_or_else(|| Error::NotFound(not_found.clone()))?; + TBlock::new(header, body) + }, + }) + } + + /// Get a pretty-printed extrinsic. + pub fn extrinsic( + &self, + input: ExtrinsicAddress< as Hash>::Output, NumberFor>, + ) -> Result { + struct ExtrinsicPrinter<'a, A: Block, B>(A::Extrinsic, &'a B); + impl<'a, A: Block, B: PrettyPrinter> fmt::Display for ExtrinsicPrinter<'a, A, B> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + self.1.fmt_extrinsic(fmt, &self.0) + } + } + + let ext = match input { + ExtrinsicAddress::Block(block, index) => { + let block = self.get_block(block)?; + block.extrinsics() + .get(index) + .cloned() + .ok_or_else(|| Error::NotFound(format!( + "Could not find extrinsic {} in block {:?}", index, block + )))? + }, + ExtrinsicAddress::Bytes(bytes) => { + TBlock::Extrinsic::decode(&mut &*bytes)? + } + }; + + Ok(format!("{}", ExtrinsicPrinter(ext, &self.printer))) + } +} diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index 5fc6ce8f101bdb807481917dcd65e679ab851cb5..d82fc49059debd1af19b4f27ab079ffe8eb0981e 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "node-primitives" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] -sp-serializer = { version = "2.0.0", path = "../../../primitives/serializer" } +sp-serializer = { version = "2.0.0-alpha.1", path = "../../../primitives/serializer" } pretty_assertions = "0.6.1" [features] diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml index 0d8106dceed7fff28e6d7bf1f7006a77362af920..029f79695a500b2aeb833c05963942ee626de8ff 100644 --- a/bin/node/rpc-client/Cargo.toml +++ b/bin/node/rpc-client/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "node-rpc-client" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] env_logger = "0.7.0" @@ -11,5 +13,5 @@ futures = "0.1.29" hyper = "0.12.35" jsonrpc-core-client = { version = "14.0.3", features = ["http", "ws"] } log = "0.4.8" -node-primitives = { version = "2.0.0", path = "../primitives" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } +node-primitives = { version = "2.0.0-alpha.1", path = "../primitives" } +sc-rpc = { version = "2.0.0-alpha.1", path = "../../../client/rpc" } diff --git a/bin/node/rpc-client/src/main.rs b/bin/node/rpc-client/src/main.rs index b28c9f8ff63922601934f90569bbd79028aa5b44..c547d30002d795b07ffc9468027bf1c06f17b877 100644 --- a/bin/node/rpc-client/src/main.rs +++ b/bin/node/rpc-client/src/main.rs @@ -19,7 +19,7 @@ //! Example substrate RPC client code. //! //! This module shows how you can write a Rust RPC client that connects to a running -//! substrate node and use staticly typed RPC wrappers. +//! substrate node and use statically typed RPC wrappers. use futures::Future; use hyper::rt; @@ -55,7 +55,7 @@ fn main() { /// 1. Calls the `pending_extrinsics` method to get all extrinsics in the pool. /// 2. Then calls `remove_extrinsic` passing the obtained raw extrinsics. /// -/// As the resul of running the code the entire content of the transaction pool is going +/// As the result of running the code the entire content of the transaction pool is going /// to be removed and the extrinsics are going to be temporarily banned. fn remove_all_extrinsics(client: AuthorClient) -> impl Future { client.pending_extrinsics() diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index 289d20ea018ac46c5d5c729dd97067aa334168b5..88dd8b7e826d4db072887e97265c453e655369d9 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -1,18 +1,27 @@ [package] name = "node-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-client = { version = "0.8", path = "../../../client/" } +sc-client = { version = "0.8.0-alpha.1", path = "../../../client/" } jsonrpc-core = "14.0.3" -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -pallet-contracts-rpc = { version = "0.8.0", path = "../../../frame/contracts/rpc/" } -pallet-transaction-payment-rpc = { version = "2.0.0", path = "../../../frame/transaction-payment/rpc/" } -substrate-frame-rpc-system = { version = "2.0.0", path = "../../../utils/frame/rpc/system" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +node-primitives = { version = "2.0.0-alpha.1", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.1", path = "../runtime" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +pallet-contracts-rpc = { version = "0.8.0-alpha.1", path = "../../../frame/contracts/rpc/" } +pallet-transaction-payment-rpc = { version = "2.0.0-alpha.1", path = "../../../frame/transaction-payment/rpc/" } +substrate-frame-rpc-system = { version = "2.0.0-alpha.1", path = "../../../utils/frame/rpc/system" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../primitives/transaction-pool" } +sc-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../client/consensus/babe" } +sc-consensus-babe-rpc = { version = "0.8.0-alpha.1", path = "../../../client/consensus/babe/rpc" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/babe" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../../client/keystore" } +sc-consensus-epochs = { version = "0.8.0-alpha.1", path = "../../../client/consensus/epochs" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 4bf0338088a9f0cc836a0b3849d14115d5058d09..4e1cfa56733a7bbbbb50fd944241410465d488c1 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -23,79 +23,136 @@ //! need some strong assumptions about the particular runtime. //! //! The RPCs available in this crate however can make some assumptions -//! about how the runtime is constructed and what `SRML` modules +//! about how the runtime is constructed and what FRAME pallets //! are part of it. Therefore all node-runtime-specific RPCs can -//! be placed here or imported from corresponding `SRML` RPC definitions. +//! be placed here or imported from corresponding FRAME RPC definitions. #![warn(missing_docs)] -use std::sync::Arc; +use std::{sync::Arc, fmt}; use node_primitives::{Block, BlockNumber, AccountId, Index, Balance}; use node_runtime::UncheckedExtrinsic; use sp_api::ProvideRuntimeApi; use sp_transaction_pool::TransactionPool; +use sp_blockchain::{Error as BlockChainError, HeaderMetadata, HeaderBackend}; +use sp_consensus::SelectChain; +use sc_keystore::KeyStorePtr; +use sp_consensus_babe::BabeApi; +use sc_consensus_epochs::SharedEpochChanges; +use sc_consensus_babe::{Config, Epoch}; +use sc_consensus_babe_rpc::BabeRPCHandler; /// Light client extra dependencies. -pub struct LightDeps { +pub struct LightDeps { + /// The client instance to use. + pub client: Arc, + /// Transaction pool instance. + pub pool: Arc

, /// Remote access to the blockchain (async). pub remote_blockchain: Arc>, /// Fetcher instance. pub fetcher: Arc, } -impl LightDeps { - /// Create empty `LightDeps` with given `F` type. - /// - /// This is a convenience method to be used in the service builder, - /// to make sure the type of the `LightDeps` is matching. - pub fn none(_: Option>) -> Option { - None - } +/// Extra dependencies for BABE. +pub struct BabeDeps { + /// BABE protocol config. + pub babe_config: Config, + /// BABE pending epoch changes. + pub shared_epoch_changes: SharedEpochChanges, + /// The keystore that manages the keys of the node. + pub keystore: KeyStorePtr, } -/// Instantiate all RPC extensions. -/// -/// If you provide `LightDeps`, the system is configured for light client. -pub fn create( - client: Arc, - pool: Arc

, - light_deps: Option>, +/// Full client dependencies. +pub struct FullDeps { + /// The client instance to use. + pub client: Arc, + /// Transaction pool instance. + pub pool: Arc

, + /// The SelectChain Strategy + pub select_chain: SC, + /// BABE specific dependencies. + pub babe: BabeDeps, +} + +/// Instantiate all Full RPC extensions. +pub fn create_full( + deps: FullDeps, ) -> jsonrpc_core::IoHandler where C: ProvideRuntimeApi, - C: sc_client::blockchain::HeaderBackend, + C: HeaderBackend + HeaderMetadata + 'static, C: Send + Sync + 'static, C::Api: substrate_frame_rpc_system::AccountNonceApi, C::Api: pallet_contracts_rpc::ContractsRuntimeApi, C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, - F: sc_client::light::fetcher::Fetcher + 'static, + C::Api: BabeApi, + ::Error: fmt::Debug, P: TransactionPool + 'static, M: jsonrpc_core::Metadata + Default, + SC: SelectChain +'static, { - use substrate_frame_rpc_system::{FullSystem, LightSystem, SystemApi}; + use substrate_frame_rpc_system::{FullSystem, SystemApi}; use pallet_contracts_rpc::{Contracts, ContractsApi}; use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi}; let mut io = jsonrpc_core::IoHandler::default(); + let FullDeps { + client, + pool, + select_chain, + babe + } = deps; + let BabeDeps { + keystore, + babe_config, + shared_epoch_changes, + } = babe; + + io.extend_with( + SystemApi::to_delegate(FullSystem::new(client.clone(), pool)) + ); + // Making synchronous calls in light client freezes the browser currently, + // more context: https://github.com/paritytech/substrate/pull/3480 + // These RPCs should use an asynchronous caller instead. + io.extend_with( + ContractsApi::to_delegate(Contracts::new(client.clone())) + ); + io.extend_with( + TransactionPaymentApi::to_delegate(TransactionPayment::new(client.clone())) + ); + io.extend_with( + sc_consensus_babe_rpc::BabeApi::to_delegate( + BabeRPCHandler::new(client, shared_epoch_changes, keystore, babe_config, select_chain) + ) + ); + + io +} + +/// Instantiate all Light RPC extensions. +pub fn create_light( + deps: LightDeps, +) -> jsonrpc_core::IoHandler where + C: sc_client::blockchain::HeaderBackend, + C: Send + Sync + 'static, + F: sc_client::light::fetcher::Fetcher + 'static, + P: TransactionPool + 'static, + M: jsonrpc_core::Metadata + Default, +{ + use substrate_frame_rpc_system::{LightSystem, SystemApi}; + + let LightDeps { + client, + pool, + remote_blockchain, + fetcher + } = deps; + let mut io = jsonrpc_core::IoHandler::default(); + io.extend_with( + SystemApi::::to_delegate(LightSystem::new(client, remote_blockchain, fetcher, pool)) + ); - if let Some(LightDeps { remote_blockchain, fetcher }) = light_deps { - io.extend_with( - SystemApi::::to_delegate(LightSystem::new(client, remote_blockchain, fetcher, pool)) - ); - } else { - io.extend_with( - SystemApi::to_delegate(FullSystem::new(client.clone(), pool)) - ); - - // Making synchronous calls in light client freezes the browser currently, - // more context: https://github.com/paritytech/substrate/pull/3480 - // These RPCs should use an asynchronous caller instead. - io.extend_with( - ContractsApi::to_delegate(Contracts::new(client.clone())) - ); - io.extend_with( - TransactionPaymentApi::to_delegate(TransactionPayment::new(client)) - ); - } io } diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 3f8e8b6731477e4cbaabaf4e3012bb139d259e6d..6e2fd1fdb3973bd63413f6f7e5689190c022283b 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "node-runtime" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] @@ -15,63 +17,64 @@ rustc-hex = { version = "2.0", optional = true } serde = { version = "1.0.102", optional = true } # primitives -sp-authority-discovery = { version = "2.0.0", default-features = false, path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8", default-features = false, path = "../../../primitives/consensus/babe" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false} -sp-inherents = { version = "2.0.0", default-features = false, path = "../../../primitives/inherents" } -node-primitives = { version = "2.0.0", default-features = false, path = "../primitives" } -sp-offchain = { version = "2.0.0", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-api = { version = "2.0.0", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "2.0.0", default-features = false, path = "../../../primitives/staking" } -sp-keyring = { version = "2.0.0", optional = true, path = "../../../primitives/keyring" } -sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } -sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } +sp-authority-discovery = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-alpha.1", default-features = false, path = "../../../primitives/consensus/babe" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.1"} +sp-inherents = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/inherents" } +node-primitives = { version = "2.0.0-alpha.1", default-features = false, path = "../primitives" } +sp-offchain = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/offchain" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/std" } +sp-api = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/runtime" } +sp-staking = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/staking" } +sp-keyring = { version = "2.0.0-alpha.1", optional = true, path = "../../../primitives/keyring" } +sp-session = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/session" } +sp-transaction-pool = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/version" } # frame dependencies -frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -frame-system = { version = "2.0.0", default-features = false, path = "../../../frame/system" } -frame-system-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } -pallet-authority-discovery = { version = "2.0.0", default-features = false, path = "../../../frame/authority-discovery" } -pallet-authorship = { version = "2.0.0", default-features = false, path = "../../../frame/authorship" } -pallet-babe = { version = "2.0.0", default-features = false, path = "../../../frame/babe" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../../../frame/balances" } -pallet-collective = { version = "2.0.0", default-features = false, path = "../../../frame/collective" } -pallet-contracts = { version = "2.0.0", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "../../../frame/contracts/common/" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } -pallet-democracy = { version = "2.0.0", default-features = false, path = "../../../frame/democracy" } -pallet-elections-phragmen = { version = "2.0.0", default-features = false, path = "../../../frame/elections-phragmen" } -pallet-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../frame/finality-tracker" } -pallet-grandpa = { version = "2.0.0", default-features = false, path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0", default-features = false, path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0", default-features = false, path = "../../../frame/indices" } -pallet-identity = { version = "2.0.0", default-features = false, path = "../../../frame/identity" } -pallet-membership = { version = "2.0.0", default-features = false, path = "../../../frame/membership" } -pallet-offences = { version = "2.0.0", default-features = false, path = "../../../frame/offences" } -pallet-randomness-collective-flip = { version = "2.0.0", default-features = false, path = "../../../frame/randomness-collective-flip" } -pallet-recovery = { version = "2.0.0", default-features = false, path = "../../../frame/recovery" } -pallet-session = { version = "2.0.0", features = ["historical"], path = "../../../frame/session", default-features = false } -pallet-staking = { version = "2.0.0", features = ["migrate"], path = "../../../frame/staking", default-features = false } -pallet-staking-reward-curve = { version = "2.0.0", path = "../../../frame/staking/reward-curve" } -pallet-sudo = { version = "2.0.0", default-features = false, path = "../../../frame/sudo" } -pallet-society = { version = "2.0.0", default-features = false, path = "../../../frame/society" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../../../frame/timestamp" } -pallet-treasury = { version = "2.0.0", default-features = false, path = "../../../frame/treasury" } -pallet-utility = { version = "2.0.0", default-features = false, path = "../../../frame/utility" } -pallet-transaction-payment = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } -pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" } +frame-executive = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/executive" } +frame-benchmarking = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/benchmarking" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/system" } +frame-system-rpc-runtime-api = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +pallet-authority-discovery = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/authority-discovery" } +pallet-authorship = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/authorship" } +pallet-babe = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/babe" } +pallet-balances = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/balances" } +pallet-collective = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/collective" } +pallet-contracts = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/contracts" } +pallet-contracts-primitives = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.1", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } +pallet-democracy = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/democracy" } +pallet-elections-phragmen = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/elections-phragmen" } +pallet-finality-tracker = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/finality-tracker" } +pallet-grandpa = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/indices" } +pallet-identity = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/identity" } +pallet-membership = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/membership" } +pallet-offences = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/offences" } +pallet-randomness-collective-flip = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/randomness-collective-flip" } +pallet-recovery = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/recovery" } +pallet-session = { version = "2.0.0-alpha.1", features = ["historical"], path = "../../../frame/session", default-features = false } +pallet-staking = { version = "2.0.0-alpha.1", features = ["migrate"], path = "../../../frame/staking", default-features = false } +pallet-staking-reward-curve = { version = "2.0.0-alpha.1", path = "../../../frame/staking/reward-curve" } +pallet-sudo = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/sudo" } +pallet-society = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/society" } +pallet-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/timestamp" } +pallet-treasury = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/treasury" } +pallet-utility = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/utility" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/transaction-payment" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-vesting = { version = "2.0.0-alpha.1", default-features = false, path = "../../../frame/vesting" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../../primitives/io" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } [features] default = ["std"] @@ -115,6 +118,7 @@ std = [ "sp-session/std", "pallet-sudo/std", "frame-support/std", + "frame-benchmarking/std", "frame-system-rpc-runtime-api/std", "frame-system/std", "pallet-timestamp/std", diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index b2c880c08bbf8098e16154e99f3a797a6d27eec8..bf12492f8db029dc22b519dd52af74adcad99a7f 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -38,7 +38,7 @@ pub mod time { /// a slot being empty). /// This value is only used indirectly to define the unit constants below /// that are expressed in blocks. The rest of the code should use - /// `SLOT_DURATION` instead (like the timestamp module for calculating the + /// `SLOT_DURATION` instead (like the Timestamp pallet for calculating the /// minimum period). /// /// If using BABE with secondary slots (default) then all of the slots will diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 7b718ca21f3714b6aa2ebd858dbb916e1148b781..980f033024d6a32b567ec22ea226244626c65fbc 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -27,15 +27,17 @@ use frame_support::{ traits::{SplitTwoWays, Currency, Randomness}, }; use sp_core::u32_trait::{_1, _2, _3, _4}; -use node_primitives::{AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, Moment, Signature}; +pub use node_primitives::{AccountId, Signature}; +use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ - Permill, Perbill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, create_runtime_str + Permill, Perbill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, create_runtime_str, }; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::transaction_validity::TransactionValidity; use sp_runtime::traits::{ - self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys, + self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, + ConvertInto, OpaqueKeys, }; use sp_version::RuntimeVersion; #[cfg(any(feature = "std", test))] @@ -79,7 +81,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 214, + spec_version: 225, impl_version: 1, apis: RUNTIME_API_VERSIONS, }; @@ -127,6 +129,9 @@ impl frame_system::Trait for Runtime { type AvailableBlockRatio = AvailableBlockRatio; type Version = Version; type ModuleToIndex = ModuleToIndex; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); } parameter_types! { @@ -157,27 +162,27 @@ impl pallet_babe::Trait for Runtime { type EpochChangeTrigger = pallet_babe::ExternalTrigger; } +parameter_types! { + pub const IndexDeposit: Balance = 1 * DOLLARS; +} + impl pallet_indices::Trait for Runtime { type AccountIndex = AccountIndex; - type IsDeadAccount = Balances; - type ResolveHint = pallet_indices::SimpleResolveHint; type Event = Event; + type Currency = Balances; + type Deposit = IndexDeposit; } parameter_types! { pub const ExistentialDeposit: Balance = 1 * DOLLARS; - pub const CreationFee: Balance = 1 * CENTS; } impl pallet_balances::Trait for Runtime { type Balance = Balance; - type OnReapAccount = ((((System, Staking), Contracts), Session), Recovery); - type OnNewAccount = Indices; - type Event = Event; type DustRemoval = (); - type TransferPayment = (); + type Event = Event; type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = frame_system::Module; } parameter_types! { @@ -232,13 +237,13 @@ parameter_types! { } impl pallet_session::Trait for Runtime { - type SessionManager = Staking; - type SessionHandler = ::KeyTypeIdProviders; - type ShouldEndSession = Babe; type Event = Event; - type Keys = SessionKeys; type ValidatorId = ::AccountId; type ValidatorIdOf = pallet_staking::StashOf; + type ShouldEndSession = Babe; + type SessionManager = Staking; + type SessionHandler = ::KeyTypeIdProviders; + type Keys = SessionKeys; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; } @@ -300,7 +305,6 @@ impl pallet_democracy::Trait for Runtime { type EnactmentPeriod = EnactmentPeriod; type LaunchPeriod = LaunchPeriod; type VotingPeriod = VotingPeriod; - type EmergencyVotingPeriod = EmergencyVotingPeriod; type MinimumDeposit = MinimumDeposit; /// A straight majority of the council can decide what their next motion is. type ExternalOrigin = pallet_collective::EnsureProportionAtLeast<_1, _2, AccountId, CouncilCollective>; @@ -312,6 +316,7 @@ impl pallet_democracy::Trait for Runtime { /// Two thirds of the technical committee can have an ExternalMajority/ExternalDefault vote /// be tabled immediately and with a shorter voting/enactment period. type FastTrackOrigin = pallet_collective::EnsureProportionAtLeast<_2, _3, AccountId, TechnicalCollective>; + type EmergencyVotingPeriod = EmergencyVotingPeriod; // To cancel a proposal which has been passed, 2/3 of the council must agree to it. type CancellationOrigin = pallet_collective::EnsureProportionAtLeast<_2, _3, AccountId, CouncilCollective>; // Any single technical committee member may veto a coming council proposal, however they can @@ -340,16 +345,16 @@ parameter_types! { impl pallet_elections_phragmen::Trait for Runtime { type Event = Event; type Currency = Balances; + type ChangeMembers = Council; type CurrencyToVote = CurrencyToVoteHandler; type CandidacyBond = CandidacyBond; type VotingBond = VotingBond; - type TermDuration = TermDuration; - type DesiredMembers = DesiredMembers; - type DesiredRunnersUp = DesiredRunnersUp; type LoserCandidate = (); type BadReport = (); type KickedMember = (); - type ChangeMembers = Council; + type DesiredMembers = DesiredMembers; + type DesiredRunnersUp = DesiredRunnersUp; + type TermDuration = TermDuration; } type TechnicalCollective = pallet_collective::Instance2; @@ -384,22 +389,20 @@ impl pallet_treasury::Trait for Runtime { type Currency = Balances; type ApproveOrigin = pallet_collective::EnsureMembers<_4, AccountId, CouncilCollective>; type RejectOrigin = pallet_collective::EnsureMembers<_2, AccountId, CouncilCollective>; + type Tippers = Elections; + type TipCountdown = TipCountdown; + type TipFindersFee = TipFindersFee; + type TipReportDepositBase = TipReportDepositBase; + type TipReportDepositPerByte = TipReportDepositPerByte; type Event = Event; type ProposalRejection = (); type ProposalBond = ProposalBond; type ProposalBondMinimum = ProposalBondMinimum; type SpendPeriod = SpendPeriod; type Burn = Burn; - type Tippers = Elections; - type TipCountdown = TipCountdown; - type TipFindersFee = TipFindersFee; - type TipReportDepositBase = TipReportDepositBase; - type TipReportDepositPerByte = TipReportDepositPerByte; } parameter_types! { - pub const ContractTransferFee: Balance = 1 * CENTS; - pub const ContractCreationFee: Balance = 1 * CENTS; pub const ContractTransactionBaseFee: Balance = 1 * CENTS; pub const ContractTransactionByteFee: Balance = 10 * MILLICENTS; pub const ContractFee: Balance = 1 * CENTS; @@ -415,7 +418,7 @@ impl pallet_contracts::Trait for Runtime { type Randomness = RandomnessCollectiveFlip; type Call = Call; type Event = Event; - type DetermineContractAddress = pallet_contracts::SimpleAddressDeterminator; + type DetermineContractAddress = pallet_contracts::SimpleAddressDeterminer; type ComputeDispatchFee = pallet_contracts::DefaultDispatchFeeComputor; type TrieIdGenerator = pallet_contracts::TrieIdFromParentCounter; type GasPayment = (); @@ -426,7 +429,6 @@ impl pallet_contracts::Trait for Runtime { type RentByteFee = RentByteFee; type RentDepositOffset = RentDepositOffset; type SurchargeReward = SurchargeReward; - type CreationFee = ContractCreationFee; type TransactionBaseFee = ContractTransactionBaseFee; type TransactionByteFee = ContractTransactionByteFee; type ContractFee = ContractFee; @@ -439,7 +441,7 @@ impl pallet_contracts::Trait for Runtime { impl pallet_sudo::Trait for Runtime { type Event = Event; - type Proposal = Call; + type Call = Call; } /// A runtime transaction submitter. @@ -451,11 +453,11 @@ parameter_types! { impl pallet_im_online::Trait for Runtime { type AuthorityId = ImOnlineId; - type Call = Call; type Event = Event; + type Call = Call; type SubmitTransaction = SubmitTransaction; - type ReportUnresponsiveness = Offences; type SessionDuration = SessionDuration; + type ReportUnresponsiveness = Offences; } impl pallet_offences::Trait for Runtime { @@ -492,14 +494,14 @@ parameter_types! { impl pallet_identity::Trait for Runtime { type Event = Event; type Currency = Balances; - type Slashed = Treasury; type BasicDeposit = BasicDeposit; type FieldDeposit = FieldDeposit; type SubAccountDeposit = SubAccountDeposit; type MaxSubAccounts = MaxSubAccounts; type MaxAdditionalFields = MaxAdditionalFields; - type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + type Slashed = Treasury; type ForceOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; } impl frame_system::offchain::CreateTransaction for Runtime { @@ -597,15 +599,15 @@ construct_runtime!( NodeBlock = node_primitives::Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: frame_system::{Module, Call, Storage, Config, Event}, + System: frame_system::{Module, Call, Config, Storage, Event}, Utility: pallet_utility::{Module, Call, Storage, Event}, Babe: pallet_babe::{Module, Call, Storage, Config, Inherent(Timestamp)}, Timestamp: pallet_timestamp::{Module, Call, Storage, Inherent}, Authorship: pallet_authorship::{Module, Call, Storage, Inherent}, - Indices: pallet_indices, - Balances: pallet_balances, + Indices: pallet_indices::{Module, Call, Storage, Config, Event}, + Balances: pallet_balances::{Module, Call, Storage, Config, Event}, TransactionPayment: pallet_transaction_payment::{Module, Storage}, - Staking: pallet_staking, + Staking: pallet_staking::{Module, Call, Config, Storage, Event}, Session: pallet_session::{Module, Call, Storage, Event, Config}, Democracy: pallet_democracy::{Module, Call, Storage, Config, Event}, Council: pallet_collective::::{Module, Call, Storage, Origin, Event, Config}, @@ -615,8 +617,8 @@ construct_runtime!( FinalityTracker: pallet_finality_tracker::{Module, Call, Inherent}, Grandpa: pallet_grandpa::{Module, Call, Storage, Config, Event}, Treasury: pallet_treasury::{Module, Call, Storage, Config, Event}, - Contracts: pallet_contracts, - Sudo: pallet_sudo, + Contracts: pallet_contracts::{Module, Call, Config, Storage, Event}, + Sudo: pallet_sudo::{Module, Call, Config, Storage, Event}, ImOnline: pallet_im_online::{Module, Call, Storage, Event, ValidateUnsigned, Config}, AuthorityDiscovery: pallet_authority_discovery::{Module, Call, Config}, Offences: pallet_offences::{Module, Call, Storage, Event}, @@ -683,6 +685,10 @@ impl_runtime_apis! { Executive::apply_extrinsic(extrinsic) } + fn apply_trusted_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResult { + Executive::apply_trusted_extrinsic(extrinsic) + } + fn finalize_block() -> ::Header { Executive::finalize_block() } @@ -734,6 +740,10 @@ impl_runtime_apis! { secondary_slots: true, } } + + fn current_epoch_start() -> sp_consensus_babe::SlotNumber { + Babe::current_epoch_start() + } } impl sp_authority_discovery::AuthorityDiscoveryApi for Runtime { @@ -804,6 +814,24 @@ impl_runtime_apis! { SessionKeys::decode_into_raw_public_keys(&encoded) } } + + impl frame_benchmarking::Benchmark for Runtime { + fn dispatch_benchmark( + module: Vec, + extrinsic: Vec, + steps: Vec, + repeat: u32, + ) -> Option> { + use frame_benchmarking::Benchmarking; + + match module.as_slice() { + b"pallet-balances" | b"balances" => Balances::run_benchmark(extrinsic, steps, repeat).ok(), + b"pallet-identity" | b"identity" => Identity::run_benchmark(extrinsic, steps, repeat).ok(), + b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(extrinsic, steps, repeat).ok(), + _ => None, + } + } + } } #[cfg(test)] diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index 051460e839eea5e2aa8e17ff76ea339809889d4c..0187fc83be20e33a20fab34652b013d35b313bd7 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -1,33 +1,57 @@ [package] name = "node-testing" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] description = "Test utilities for Substrate node." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +publish = true [dependencies] -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -sc-client = { version = "0.8", path = "../../../client/" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../../../frame/balances" } +sc-client = { version = "0.8.0-alpha.1", path = "../../../client/" } +sc-client-db = { version = "0.8.0-alpha.1", path = "../../../client/db/", features = ["kvdb-rocksdb"] } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../../client/api/" } codec = { package = "parity-scale-codec", version = "1.0.0" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0", path = "../../../frame/grandpa" } -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -node-executor = { version = "2.0.0", path = "../executor" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -frame-support = { version = "2.0.0", path = "../../../frame/support" } -pallet-session = { version = "2.0.0", path = "../../../frame/session" } -pallet-society = { version = "2.0.0", path = "../../../frame/society" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -pallet-staking = { version = "2.0.0", path = "../../../frame/staking" } -sc-executor = { version = "0.8", path = "../../../client/executor" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } -pallet-timestamp = { version = "2.0.0", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0", path = "../../../frame/treasury" } +pallet-contracts = { version = "2.0.0-alpha.1", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-alpha.1", path = "../../../frame/grandpa" } +pallet-indices = { version = "2.0.0-alpha.1", path = "../../../frame/indices" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../../primitives/keyring" } +node-executor = { version = "2.0.0-alpha.1", path = "../executor" } +node-primitives = { version = "2.0.0-alpha.1", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.1", path = "../runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } +frame-support = { version = "2.0.0-alpha.1", path = "../../../frame/support" } +pallet-session = { version = "2.0.0-alpha.1", path = "../../../frame/session" } +pallet-society = { version = "2.0.0-alpha.1", path = "../../../frame/society" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +pallet-staking = { version = "2.0.0-alpha.1", path = "../../../frame/staking" } +sc-executor = { version = "0.8.0-alpha.1", path = "../../../client/executor", features = ["wasmtime"] } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +frame-system = { version = "2.0.0-alpha.1", path = "../../../frame/system" } +substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } +pallet-timestamp = { version = "2.0.0-alpha.1", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-alpha.1", path = "../../../frame/treasury" } wabt = "0.9.2" +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sp-finality-tracker = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/timestamp" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +log = "0.4.8" +tempfile = "3.1.0" +fs_extra = "1" + +[dev-dependencies] +criterion = "0.3.0" +sc-cli = { version = "0.8.0-alpha.1", path = "../../../client/cli" } +sc-service = { version = "0.8.0-alpha.1", path = "../../../client/service", features = ["rocksdb"] } + +[[bench]] +name = "import" +harness = false diff --git a/bin/node/testing/benches/import.rs b/bin/node/testing/benches/import.rs new file mode 100644 index 0000000000000000000000000000000000000000..79cb71b164371e3bd6bb29c0f58ce2efa28a59af --- /dev/null +++ b/bin/node/testing/benches/import.rs @@ -0,0 +1,205 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block import benchmark. +//! +//! This benchmark is expected to measure block import operation of +//! some more or less full block. +//! +//! As we also want to protect against cold-cache attacks, this +//! benchmark should not rely on any caching (except those that +//! DO NOT depend on user input). Thus block generation should be +//! based on randomized operation. +//! +//! This is supposed to be very simple benchmark and is not subject +//! to much configuring - just block full of randomized transactions. +//! It is not supposed to measure runtime modules weight correctness + +use std::fmt; +use node_testing::bench::{BenchDb, Profile}; +use node_primitives::Block; +use sp_runtime::generic::BlockId; +use criterion::{Criterion, criterion_group, criterion_main}; +use sc_client_api::backend::Backend; + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(20).warm_up_time(std::time::Duration::from_secs(20)); + targets = bench_block_import +); +criterion_group!( + name = wasm_size; + config = Criterion::default().sample_size(10); + targets = bench_wasm_size_import +); +criterion_group!( + name = profile; + config = Criterion::default().sample_size(10); + targets = profile_block_import +); +criterion_main!(benches, profile); + +fn bench_block_import(c: &mut Criterion) { + sc_cli::init_logger(""); + // for future uses, uncomment if something wrong. + // sc_cli::init_logger("sc_client=debug"); + + let mut bench_db = BenchDb::new(128); + let block = bench_db.generate_block(100); + + log::trace!( + target: "bench-logistics", + "Seed database directory: {}", + bench_db.path().display(), + ); + + c.bench_function_over_inputs("import block", + move |bencher, profile| { + bencher.iter_batched( + || { + let context = bench_db.create_context(*profile); + + // mostly to just launch compiler before benching! + let version = context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version") + .spec_version; + + log::trace!( + target: "bench-logistics", + "Next iteration database directory: {}, runtime version: {}", + context.path().display(), version, + ); + + context + }, + |mut context| { + let start = std::time::Instant::now(); + context.import_block(block.clone()); + let elapsed = start.elapsed(); + + log::info!( + target: "bench-logistics", + "imported block with {} tx, took: {:#?}", + block.extrinsics.len(), + elapsed, + ); + + log::info!( + target: "bench-logistics", + "usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ); + }, + criterion::BatchSize::PerIteration, + ); + }, + vec![Profile::Wasm, Profile::Native], + ); +} + +// This is not an actual benchmark, so don't use it to measure anything. +// It just produces special pattern of cpu load that allows easy picking +// the part of block import for the profiling in the tool of choice. +fn profile_block_import(c: &mut Criterion) { + sc_cli::init_logger(""); + + let mut bench_db = BenchDb::new(128); + let block = bench_db.generate_block(100); + + c.bench_function("profile block", + move |bencher| { + bencher.iter_batched( + || { + bench_db.create_context(Profile::Native) + }, + |mut context| { + // until better osx signpost/callgrind signal is possible to use + // in rust, we just pause everything completely to help choosing + // actual profiling interval + std::thread::park_timeout(std::time::Duration::from_secs(2)); + context.import_block(block.clone()); + // and here as well + std::thread::park_timeout(std::time::Duration::from_secs(2)); + log::info!( + target: "bench-logistics", + "imported block, usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ) + }, + criterion::BatchSize::PerIteration, + ); + }, + ); +} + +struct Setup { + db: BenchDb, + block: Block, +} + +struct SetupIterator { + current: usize, + finish: usize, + multiplier: usize, +} + +impl SetupIterator { + fn new(current: usize, finish: usize, multiplier: usize) -> Self { + SetupIterator { current, finish, multiplier } + } +} + +impl Iterator for SetupIterator { + type Item = Setup; + + fn next(&mut self) -> Option { + if self.current >= self.finish { return None } + + self.current += 1; + + let size = self.current * self.multiplier; + let mut db = BenchDb::new(size); + let block = db.generate_block(size); + Some(Setup { db, block }) + } +} + +impl fmt::Debug for Setup { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Setup: {} tx/block", self.block.extrinsics.len()) + } +} + +fn bench_wasm_size_import(c: &mut Criterion) { + sc_cli::init_logger(""); + + c.bench_function_over_inputs("wasm_size_import", + move |bencher, setup| { + bencher.iter_batched( + || { + setup.db.create_context(Profile::Wasm) + }, + |mut context| { + context.import_block(setup.block.clone()); + }, + criterion::BatchSize::PerIteration, + ); + }, + SetupIterator::new(5, 15, 50), + ); +} diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs new file mode 100644 index 0000000000000000000000000000000000000000..58a7ab933eeb9d55f31bd1b9e45d91b69a4c40c4 --- /dev/null +++ b/bin/node/testing/src/bench.rs @@ -0,0 +1,424 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Benchmarking module. +//! +//! Utilities to do full-scale benchmarks involving database. With `BenchDb` you +//! can pregenerate seed database and `clone` it for every iteration of your benchmarks +//! or tests to get consistent, smooth benchmark experience! + +use std::{sync::Arc, path::Path, collections::BTreeMap}; + +use node_primitives::Block; +use crate::client::{Client, Backend}; +use crate::keyring::*; +use sc_client_db::PruningMode; +use sc_executor::{NativeExecutor, WasmExecutionMethod}; +use sp_consensus::{ + BlockOrigin, BlockImport, BlockImportParams, + ForkChoiceStrategy, ImportResult, ImportedAux +}; +use sp_runtime::{ + generic::BlockId, + OpaqueExtrinsic, + traits::{Block as BlockT, Verify, Zero, IdentifyAccount}, +}; +use codec::{Decode, Encode}; +use node_runtime::{ + Call, + CheckedExtrinsic, + constants::currency::DOLLARS, + UncheckedExtrinsic, + MinimumPeriod, + BalancesCall, + AccountId, + Signature, +}; +use sp_core::ExecutionContext; +use sp_api::ProvideRuntimeApi; +use sp_block_builder::BlockBuilder; +use sp_inherents::InherentData; +use sc_client_api::{ + ExecutionStrategy, + execution_extensions::{ExecutionExtensions, ExecutionStrategies}, +}; +use sp_core::{Pair, Public, sr25519}; + +/// Keyring full of accounts for benching. +/// +/// Accounts are ordered: +/// //endowed-user//00 +/// //endowed-user//01 +/// ... +/// //endowed-user//N +#[derive(Clone)] +pub struct BenchKeyring { + accounts: BTreeMap, +} + +/// Pre-initialized benchmarking database. +/// +/// This is prepared database with genesis and keyring +/// that can be cloned and then used for any benchmarking. +pub struct BenchDb { + keyring: BenchKeyring, + directory_guard: Guard, +} + +impl Clone for BenchDb { + fn clone(&self) -> Self { + let keyring = self.keyring.clone(); + let dir = tempfile::tempdir().expect("temp dir creation failed"); + + let seed_dir = self.directory_guard.0.path(); + + log::trace!( + target: "bench-logistics", + "Copying seed db from {} to {}", + seed_dir.to_string_lossy(), + dir.path().to_string_lossy(), + ); + let seed_db_files = std::fs::read_dir(seed_dir) + .expect("failed to list file in seed dir") + .map(|f_result| + f_result.expect("failed to read file in seed db") + .path() + .clone() + ).collect(); + fs_extra::copy_items( + &seed_db_files, + dir.path(), + &fs_extra::dir::CopyOptions::new(), + ).expect("Copy of seed database is ok"); + + BenchDb { keyring, directory_guard: Guard(dir) } + } +} + +impl BenchDb { + /// New immutable benchmarking database. + /// + /// This will generate database files in random temporary directory + /// and keep it there until struct is dropped. + /// + /// You can `clone` this database or you can `create_context` from it + /// (which also do `clone`) to run actual operation against new database + /// which will be identical to this. + pub fn new(keyring_length: usize) -> Self { + let keyring = BenchKeyring::new(keyring_length); + + let dir = tempfile::tempdir().expect("temp dir creation failed"); + log::trace!( + target: "bench-logistics", + "Created seed db at {}", + dir.path().to_string_lossy(), + ); + let (_client, _backend) = Self::bench_client(dir.path(), Profile::Native, &keyring); + let directory_guard = Guard(dir); + + BenchDb { keyring, directory_guard } + } + + // This should return client that is doing everything that full node + // is doing. + // + // - This client should use best wasm execution method. + // - This client should work with real database only. + fn bench_client(dir: &std::path::Path, profile: Profile, keyring: &BenchKeyring) -> (Client, std::sync::Arc) { + let db_config = sc_client_db::DatabaseSettings { + state_cache_size: 16*1024*1024, + state_cache_child_ratio: Some((0, 100)), + pruning: PruningMode::ArchiveAll, + source: sc_client_db::DatabaseSettingsSrc::Path { + path: dir.into(), + cache_size: None, + }, + }; + + let (client, backend) = sc_client_db::new_client( + db_config, + NativeExecutor::new(WasmExecutionMethod::Compiled, None), + &keyring.generate_genesis(), + None, + None, + ExecutionExtensions::new(profile.into_execution_strategies(), None), + ).expect("Should not fail"); + + (client, backend) + } + + /// Generate new block using this database. + pub fn generate_block(&mut self, transactions: usize) -> Block { + let (client, _backend) = Self::bench_client( + self.directory_guard.path(), + Profile::Wasm, + &self.keyring, + ); + + let version = client.runtime_version_at(&BlockId::number(0)) + .expect("There should be runtime version at 0") + .spec_version; + + let genesis_hash = client.block_hash(Zero::zero()) + .expect("Database error?") + .expect("Genesis block always exists; qed") + .into(); + + let mut block = client + .new_block(Default::default()) + .expect("Block creation failed"); + + let timestamp = 1 * MinimumPeriod::get(); + + let mut inherent_data = InherentData::new(); + inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) + .expect("Put timestamp failed"); + inherent_data.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &0) + .expect("Put finality tracker failed"); + + for extrinsic in client.runtime_api() + .inherent_extrinsics_with_context( + &BlockId::number(0), + ExecutionContext::BlockConstruction, + inherent_data, + ).expect("Get inherents failed") + { + block.push(extrinsic).expect("Push inherent failed"); + } + + let mut iteration = 0; + let start = std::time::Instant::now(); + for _ in 0..transactions { + + let sender = self.keyring.at(iteration); + let receiver = get_account_id_from_seed::( + &format!("random-user//{}", iteration) + ); + + let signed = self.keyring.sign( + CheckedExtrinsic { + signed: Some((sender, signed_extra(0, 1*DOLLARS))), + function: Call::Balances( + BalancesCall::transfer( + pallet_indices::address::Address::Id(receiver), + 1*DOLLARS + ) + ), + }, + version, + genesis_hash, + ); + + let encoded = Encode::encode(&signed); + + let opaque = OpaqueExtrinsic::decode(&mut &encoded[..]) + .expect("Failed to decode opaque"); + + match block.push(opaque) { + Err(sp_blockchain::Error::ApplyExtrinsicFailed( + sp_blockchain::ApplyExtrinsicFailed::Validity(e) + )) if e.exhausted_resources() => { + break; + }, + Err(err) => panic!("Error pushing transaction: {:?}", err), + Ok(_) => {}, + } + iteration += 1; + } + let block = block.build().expect("Block build failed").block; + + log::info!( + target: "bench-logistics", + "Block construction: {:#?} ({} tx)", + start.elapsed(), block.extrinsics.len() + ); + + block + } + + /// Database path. + pub fn path(&self) -> &Path { + self.directory_guard.path() + } + + /// Clone this database and create context for testing/benchmarking. + pub fn create_context(&self, profile: Profile) -> BenchContext { + let BenchDb { directory_guard, keyring } = self.clone(); + let (client, backend) = Self::bench_client(directory_guard.path(), profile, &keyring); + + BenchContext { + client, backend, db_guard: directory_guard, + } + } +} + +impl BenchKeyring { + /// New keyring. + /// + /// `length` is the number of accounts generated. + pub fn new(length: usize) -> Self { + let mut accounts = BTreeMap::new(); + + for n in 0..length { + let seed = format!("//endowed-user/{}", n); + let pair = sr25519::Pair::from_string(&seed, None).expect("failed to generate pair"); + let account_id = AccountPublic::from(pair.public()).into_account(); + accounts.insert(account_id, pair); + } + + Self { accounts } + } + + /// Generated account id-s from keyring keypairs. + pub fn collect_account_ids(&self) -> Vec { + self.accounts.keys().cloned().collect() + } + + /// Get account id at position `index` + pub fn at(&self, index: usize) -> AccountId { + self.accounts.keys().nth(index).expect("Failed to get account").clone() + } + + /// Sign transaction with keypair from this keyring. + pub fn sign(&self, xt: CheckedExtrinsic, version: u32, genesis_hash: [u8; 32]) -> UncheckedExtrinsic { + match xt.signed { + Some((signed, extra)) => { + let payload = (xt.function, extra.clone(), version, genesis_hash, genesis_hash); + let key = self.accounts.get(&signed).expect("Account id not found in keyring"); + let signature = payload.using_encoded(|b| { + if b.len() > 256 { + key.sign(&sp_io::hashing::blake2_256(b)) + } else { + key.sign(b) + } + }).into(); + UncheckedExtrinsic { + signature: Some((pallet_indices::address::Address::Id(signed), signature, extra)), + function: payload.0, + } + } + None => UncheckedExtrinsic { + signature: None, + function: xt.function, + }, + } + } + + /// Generate genesis with accounts from this keyring endowed with some balance. + pub fn generate_genesis(&self) -> node_runtime::GenesisConfig { + crate::genesis::config_endowed( + false, + Some(node_runtime::WASM_BINARY), + self.collect_account_ids(), + ) + } +} + +/// Profile for exetion strategies. +#[derive(Clone, Copy, Debug)] +pub enum Profile { + /// As native as possible. + Native, + /// As wasm as possible. + Wasm, +} + +impl Profile { + fn into_execution_strategies(self) -> ExecutionStrategies { + match self { + Profile::Wasm => ExecutionStrategies { + syncing: ExecutionStrategy::AlwaysWasm, + importing: ExecutionStrategy::AlwaysWasm, + block_construction: ExecutionStrategy::AlwaysWasm, + offchain_worker: ExecutionStrategy::AlwaysWasm, + other: ExecutionStrategy::AlwaysWasm, + }, + Profile::Native => ExecutionStrategies { + syncing: ExecutionStrategy::NativeElseWasm, + importing: ExecutionStrategy::NativeElseWasm, + block_construction: ExecutionStrategy::NativeElseWasm, + offchain_worker: ExecutionStrategy::NativeElseWasm, + other: ExecutionStrategy::NativeElseWasm, + } + } + } +} + +struct Guard(tempfile::TempDir); + +impl Guard { + fn path(&self) -> &Path { + self.0.path() + } +} + +/// Benchmarking/test context holding instantiated client and backend references. +pub struct BenchContext { + /// Node client. + pub client: Client, + /// Node backend. + pub backend: Arc, + + db_guard: Guard, +} + +type AccountPublic = ::Signer; + +fn get_from_seed(seed: &str) -> ::Public { + TPublic::Pair::from_string(&format!("//{}", seed), None) + .expect("static values are valid; qed") + .public() +} + +fn get_account_id_from_seed(seed: &str) -> AccountId +where + AccountPublic: From<::Public> +{ + AccountPublic::from(get_from_seed::(seed)).into_account() +} + +impl BenchContext { + /// Import some block. + pub fn import_block(&mut self, block: Block) { + let mut import_params = BlockImportParams::new(BlockOrigin::NetworkBroadcast, block.header.clone()); + import_params.body = Some(block.extrinsics().to_vec()); + import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain); + + assert_eq!(self.client.chain_info().best_number, 0); + + assert_eq!( + self.client.import_block(import_params, Default::default()) + .expect("Failed to import block"), + ImportResult::Imported( + ImportedAux { + header_only: false, + clear_justification_requests: false, + needs_justification: false, + bad_justification: false, + needs_finality_proof: false, + is_new_best: true, + } + ) + ); + + assert_eq!(self.client.chain_info().best_number, 1); + } + + /// Database path for the current context. + pub fn path(&self) -> &Path { + self.db_guard.path() + } +} diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 1dddd8ba5aed8562ad6742b7a9e00ed08933ff51..963bac7041b7d8ac25686e8e872b72d9210fb23b 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Utilites to build a `TestClient` for `node-runtime`. +//! Utilities to build a `TestClient` for `node-runtime`. use sp_runtime::BuildStorage; @@ -35,6 +35,9 @@ pub type Client = sc_client::Client< node_runtime::RuntimeApi, >; +/// Transaction for node-runtime. +pub type Transaction = sc_client_api::backend::TransactionFor; + /// Genesis configuration parameters for `TestClient`. #[derive(Default)] pub struct GenesisParameters { diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 8c5514defa4d8b1d13174fb4ecba9e72a94c43f2..6aa7290a641bfb85675549531ad1238b5f813483 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -21,14 +21,38 @@ use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use node_runtime::{ GenesisConfig, BalancesConfig, SessionConfig, StakingConfig, SystemConfig, GrandpaConfig, IndicesConfig, ContractsConfig, SocietyConfig, WASM_BINARY, + AccountId, }; use node_runtime::constants::currency::*; use sp_core::ChangesTrieConfiguration; use sp_runtime::Perbill; - /// Create genesis runtime configuration for tests. pub fn config(support_changes_trie: bool, code: Option<&[u8]>) -> GenesisConfig { + config_endowed(support_changes_trie, code, Default::default()) +} + +/// Create genesis runtime configuration for tests with some extra +/// endowed accounts. +pub fn config_endowed( + support_changes_trie: bool, + code: Option<&[u8]>, + extra_endowed: Vec, +) -> GenesisConfig { + + let mut endowed = vec![ + (alice(), 111 * DOLLARS), + (bob(), 100 * DOLLARS), + (charlie(), 100_000_000 * DOLLARS), + (dave(), 111 * DOLLARS), + (eve(), 101 * DOLLARS), + (ferdie(), 100 * DOLLARS), + ]; + + endowed.extend( + extra_endowed.into_iter().map(|endowed| (endowed, 100*DOLLARS)) + ); + GenesisConfig { frame_system: Some(SystemConfig { changes_trie_config: if support_changes_trie { Some(ChangesTrieConfiguration { @@ -38,29 +62,22 @@ pub fn config(support_changes_trie: bool, code: Option<&[u8]>) -> GenesisConfig code: code.map(|x| x.to_vec()).unwrap_or_else(|| WASM_BINARY.to_vec()), }), pallet_indices: Some(IndicesConfig { - ids: vec![alice(), bob(), charlie(), dave(), eve(), ferdie()], + indices: vec![], }), pallet_balances: Some(BalancesConfig { - balances: vec![ - (alice(), 111 * DOLLARS), - (bob(), 100 * DOLLARS), - (charlie(), 100_000_000 * DOLLARS), - (dave(), 111 * DOLLARS), - (eve(), 101 * DOLLARS), - (ferdie(), 100 * DOLLARS), - ], + balances: endowed, }), pallet_session: Some(SessionConfig { keys: vec![ - (alice(), to_session_keys( + (dave(), alice(), to_session_keys( &Ed25519Keyring::Alice, &Sr25519Keyring::Alice, )), - (bob(), to_session_keys( + (eve(), bob(), to_session_keys( &Ed25519Keyring::Bob, &Sr25519Keyring::Bob, )), - (charlie(), to_session_keys( + (ferdie(), charlie(), to_session_keys( &Ed25519Keyring::Charlie, &Sr25519Keyring::Charlie, )), diff --git a/bin/node/testing/src/lib.rs b/bin/node/testing/src/lib.rs index f4a5e75e492d3dcc294276d48c970e78071dba0d..6a06d318016f290fd96d6a33fbe7c2e2163ce988 100644 --- a/bin/node/testing/src/lib.rs +++ b/bin/node/testing/src/lib.rs @@ -21,4 +21,4 @@ pub mod client; pub mod genesis; pub mod keyring; - +pub mod bench; diff --git a/bin/node/transaction-factory/Cargo.toml b/bin/node/transaction-factory/Cargo.toml index cd40e2f0d939db732339659aeea02f184b062096..b73f5b6582407d02b0ab1fc047fd8039f3fbd156 100644 --- a/bin/node/transaction-factory/Cargo.toml +++ b/bin/node/transaction-factory/Cargo.toml @@ -1,20 +1,22 @@ [package] name = "node-transaction-factory" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0", path = "../../../client/api" } -sc-client = { version = "0.8", path = "../../../client" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../../primitives/block-builder" } +sc-cli = { version = "0.8.0-alpha.1", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../../client/api" } +sc-client = { version = "0.8.0-alpha.1", path = "../../../client" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sc-service = { version = "0.8", path = "../../../client/service" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sc-service = { version = "0.8.0-alpha.1", path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } diff --git a/bin/node/transaction-factory/src/complex_mode.rs b/bin/node/transaction-factory/src/complex_mode.rs deleted file mode 100644 index 6d7e60c8d3c1f937301f34746faffdd734c26c36..0000000000000000000000000000000000000000 --- a/bin/node/transaction-factory/src/complex_mode.rs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -/// This module implements the `MasterToNToM` mode: -/// -/// Manufacture `num` transactions from the master account to `num` -/// randomly created accounts. From each of these randomly created -/// accounts manufacture a transaction to another randomly created -/// account. -/// Repeat this round `rounds` times. If `rounds` = 1 the behavior -/// is the same as `MasterToN`. -/// -/// A -> B -/// A -> C -/// A -> D -/// ... x `num` -/// -/// B -> E -/// C -> F -/// D -> G -/// ... -/// E -> H -/// F -> I -/// G -> J -/// ... -/// ... x `rounds` - -use std::sync::Arc; - -use log::info; -use sc_client::Client; -use sp_block_builder::BlockBuilder; -use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi}; -use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, One, Zero}; - -use crate::{RuntimeAdapter, create_block}; - -pub fn next( - factory_state: &mut RA, - client: &Arc>, - version: u32, - genesis_hash: ::Hash, - prior_block_hash: ::Hash, - prior_block_id: BlockId, -) -> Option -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilder + - sp_api::ApiExt, - RtApi: ConstructRuntimeApi> + Send + Sync, - RA: RuntimeAdapter, -{ - let total = factory_state.start_number() + factory_state.num() * factory_state.rounds(); - - if factory_state.block_no() >= total || factory_state.round() >= factory_state.rounds() { - return None; - } - - info!( - "Round {}: Creating {} transactions in total, {} per round. {} rounds in total.", - factory_state.round() + RA::Number::one(), - factory_state.num() * factory_state.rounds(), - factory_state.num(), - factory_state.rounds(), - ); - - let from = from::(factory_state); - - let seed = factory_state.start_number() + factory_state.block_no(); - let to = RA::gen_random_account_id(&seed); - - let rounds_left = factory_state.rounds() - factory_state.round(); - let amount = RA::minimum_balance() * rounds_left.into(); - - let transfer = factory_state.transfer_extrinsic( - &from.0, - &from.1, - &to, - &amount, - version, - &genesis_hash, - &prior_block_hash, - ); - - let inherents = factory_state.inherent_extrinsics(); - let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents) - .expect("Failed to create inherent extrinsics"); - - let block = create_block::(&client, transfer, inherents); - info!( - "Created block {} with hash {}. Transferring {} from {} to {}.", - factory_state.block_no() + RA::Number::one(), - prior_block_hash, - amount, - from.0, - to - ); - - factory_state.set_block_no(factory_state.block_no() + RA::Number::one()); - - let new_round = factory_state.block_no() > RA::Number::zero() - && factory_state.block_no() % factory_state.num() == RA::Number::zero(); - if new_round { - factory_state.set_round(factory_state.round() + RA::Number::one()); - factory_state.set_block_in_round(RA::Number::zero()); - } else { - factory_state.set_block_in_round(factory_state.block_in_round() + RA::Number::one()); - } - - Some(block) -} - -/// Return the account which received tokens at this point in the previous round. -fn from( - factory_state: &mut RA -) -> (::AccountId, ::Secret) -where RA: RuntimeAdapter -{ - let is_first_round = factory_state.round() == RA::Number::zero(); - match is_first_round { - true => { - // first round always uses master account - (RA::master_account_id(), RA::master_account_secret()) - }, - _ => { - // the account to which was sent in the last round - let is_round_one = factory_state.round() == RA::Number::one(); - let seed = match is_round_one { - true => factory_state.start_number() + factory_state.block_in_round(), - _ => { - let block_no_in_prior_round = - factory_state.num() * (factory_state.round() - RA::Number::one()) + factory_state.block_in_round(); - factory_state.start_number() + block_no_in_prior_round - } - }; - (RA::gen_random_account_id(&seed), RA::gen_random_account_secret(&seed)) - }, - } -} diff --git a/bin/node/transaction-factory/src/lib.rs b/bin/node/transaction-factory/src/lib.rs index a0c6a4f663c2adede2a4e036c71d11dd1a6c0399..a4c001145ac7eb6bc99d7fa91740fe4d265650e3 100644 --- a/bin/node/transaction-factory/src/lib.rs +++ b/bin/node/transaction-factory/src/lib.rs @@ -37,39 +37,28 @@ use sp_consensus::block_import::BlockImport; use codec::{Decode, Encode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, SimpleArithmetic, One, Zero, + Block as BlockT, Header as HeaderT, AtLeast32Bit, One, Zero, }; -pub use crate::modes::Mode; - -pub mod modes; -mod complex_mode; -mod simple_modes; pub trait RuntimeAdapter { type AccountId: Display; - type Balance: Display + SimpleArithmetic + From; + type Balance: Display + AtLeast32Bit + From; type Block: BlockT; type Index: Copy; - type Number: Display + PartialOrd + SimpleArithmetic + Zero + One; + type Number: Display + PartialOrd + AtLeast32Bit + Zero + One; type Phase: Copy; type Secret; - fn new(mode: Mode, rounds: u64, start_number: u64) -> Self; + fn new(blocks: u32, transactions: u32) -> Self; - fn block_no(&self) -> Self::Number; - fn block_in_round(&self) -> Self::Number; - fn mode(&self) -> &Mode; - fn num(&self) -> Self::Number; - fn rounds(&self) -> Self::Number; - fn round(&self) -> Self::Number; - fn start_number(&self) -> Self::Number; + fn blocks(&self) -> u32; + fn transactions(&self) -> u32; - fn set_block_in_round(&mut self, val: Self::Number); - fn set_block_no(&mut self, val: Self::Number); - fn set_round(&mut self, val: Self::Number); + fn block_number(&self) -> u32; + fn set_block_number(&mut self, value: u32); fn transfer_extrinsic( - &self, + &mut self, sender: &Self::AccountId, key: &Self::Secret, destination: &Self::AccountId, @@ -84,19 +73,17 @@ pub trait RuntimeAdapter { fn minimum_balance() -> Self::Balance; fn master_account_id() -> Self::AccountId; fn master_account_secret() -> Self::Secret; - fn extract_index(&self, account_id: &Self::AccountId, block_hash: &::Hash) -> Self::Index; - fn extract_phase(&self, block_hash: ::Hash) -> Self::Phase; - fn gen_random_account_id(seed: &Self::Number) -> Self::AccountId; - fn gen_random_account_secret(seed: &Self::Number) -> Self::Secret; + + fn gen_random_account_id(seed: u32) -> Self::AccountId; + fn gen_random_account_secret(seed: u32) -> Self::Secret; } -/// Manufactures transactions. The exact amount depends on -/// `mode`, `num` and `rounds`. +/// Manufactures transactions. The exact amount depends on `num` and `rounds`. pub fn factory( mut factory_state: RA, client: &Arc>, select_chain: &Sc, -) -> sc_cli::error::Result<()> +) -> sc_cli::Result<()> where Block: BlockT, Exec: sc_client::CallExecutor + Send + Sync + Clone, @@ -110,12 +97,7 @@ where RA: RuntimeAdapter, Block::Hash: From, { - if *factory_state.mode() != Mode::MasterToNToM && factory_state.rounds() > RA::Number::one() { - let msg = "The factory can only be used with rounds set to 1 in this mode.".into(); - return Err(sc_cli::error::Error::Input(msg)); - } - - let best_header: Result<::Header, sc_cli::error::Error> = + let best_header: Result<::Header, sc_cli::Error> = select_chain.best_chain().map_err(|e| format!("{:?}", e).into()); let mut best_hash = best_header?.hash(); let mut best_block_id = BlockId::::hash(best_hash); @@ -123,89 +105,64 @@ where let genesis_hash = client.block_hash(Zero::zero())? .expect("Genesis block always exists; qed").into(); - while let Some(block) = match factory_state.mode() { - Mode::MasterToNToM => complex_mode::next::( - &mut factory_state, - &client, - version, - genesis_hash, - best_hash.into(), - best_block_id, - ), - _ => simple_modes::next::( - &mut factory_state, - &client, - version, - genesis_hash, - best_hash.into(), - best_block_id, - ), - } { + while factory_state.block_number() < factory_state.blocks() { + let from = (RA::master_account_id(), RA::master_account_secret()); + let amount = RA::minimum_balance(); + + let inherents = RA::inherent_extrinsics(&factory_state); + let inherents = client.runtime_api().inherent_extrinsics(&best_block_id, inherents) + .expect("Failed to create inherent extrinsics"); + + let tx_per_block = factory_state.transactions(); + + let mut block = client.new_block(Default::default()).expect("Failed to create new block"); + + for tx_num in 0..tx_per_block { + let seed = tx_num * (factory_state.block_number() + 1); + let to = RA::gen_random_account_id(seed); + + let transfer = factory_state.transfer_extrinsic( + &from.0, + &from.1, + &to, + &amount, + version, + &genesis_hash, + &best_hash, + ); + + info!("Pushing transfer {}/{} to {} into block.", tx_num + 1, tx_per_block, to); + + block.push( + Decode::decode(&mut &transfer.encode()[..]) + .expect("Failed to decode transfer extrinsic") + ).expect("Failed to push transfer extrinsic into block"); + } + + for inherent in inherents { + block.push(inherent).expect("Failed ..."); + } + + let block = block.build().expect("Failed to bake block").block; + + factory_state.set_block_number(factory_state.block_number() + 1); + + info!( + "Created block {} with hash {}.", + factory_state.block_number(), + best_hash, + ); + best_hash = block.header().hash(); best_block_id = BlockId::::hash(best_hash); - import_block(client.clone(), block); - - info!("Imported block at {}", factory_state.block_no()); - } - Ok(()) -} + let mut import = BlockImportParams::new(BlockOrigin::File, block.header().clone()); + import.body = Some(block.extrinsics().to_vec()); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + client.clone().import_block(import, HashMap::new()).expect("Failed to import block"); -/// Create a baked block from a transfer extrinsic and timestamp inherent. -pub fn create_block( - client: &Arc>, - transfer: ::Extrinsic, - inherent_extrinsics: Vec<::Extrinsic>, -) -> Block -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - RtApi: ConstructRuntimeApi> + Send + Sync, - as ProvideRuntimeApi>::Api: - BlockBuilder + - ApiExt, - RA: RuntimeAdapter, -{ - let mut block = client.new_block(Default::default()).expect("Failed to create new block"); - block.push( - Decode::decode(&mut &transfer.encode()[..]) - .expect("Failed to decode transfer extrinsic") - ).expect("Failed to push transfer extrinsic into block"); - - for inherent in inherent_extrinsics { - block.push(inherent).expect("Failed ..."); + info!("Imported block at {}", factory_state.block_number()); } - block.build().expect("Failed to bake block").block -} - -fn import_block( - mut client: Arc>, - block: Block -) -> () where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - sp_api::Core + - ApiExt, -{ - let import = BlockImportParams { - origin: BlockOrigin::File, - header: block.header().clone(), - post_digests: Vec::new(), - body: Some(block.extrinsics().to_vec()), - storage_changes: None, - finalized: false, - justification: None, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; - client.import_block(import, HashMap::new()).expect("Failed to import block"); + Ok(()) } diff --git a/bin/node/transaction-factory/src/simple_modes.rs b/bin/node/transaction-factory/src/simple_modes.rs deleted file mode 100644 index fba328731a98fe44ba5a17660c8bb816147b6f80..0000000000000000000000000000000000000000 --- a/bin/node/transaction-factory/src/simple_modes.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -/// This module implements two manufacturing modes: -/// -/// # MasterToN -/// Manufacture `num` transactions from the master account -/// to `num` randomly created accounts, one each. -/// -/// A -> B -/// A -> C -/// ... x `num` -/// -/// -/// # MasterTo1 -/// Manufacture `num` transactions from the master account -/// to exactly one other randomly created account. -/// -/// A -> B -/// A -> B -/// ... x `num` - -use std::sync::Arc; - -use log::info; -use sc_client::Client; -use sp_block_builder::BlockBuilder; -use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi}; -use sp_runtime::traits::{Block as BlockT, One}; -use sp_runtime::generic::BlockId; - -use crate::{Mode, RuntimeAdapter, create_block}; - -pub fn next( - factory_state: &mut RA, - client: &Arc>, - version: u32, - genesis_hash: ::Hash, - prior_block_hash: ::Hash, - prior_block_id: BlockId, -) -> Option -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilder + - sp_api::ApiExt, - RtApi: ConstructRuntimeApi> + Send + Sync, - RA: RuntimeAdapter, -{ - if factory_state.block_no() >= factory_state.num() { - return None; - } - - let from = (RA::master_account_id(), RA::master_account_secret()); - - let seed = match factory_state.mode() { - // choose the same receiver for all transactions - Mode::MasterTo1 => factory_state.start_number(), - - // different receiver for each transaction - Mode::MasterToN => factory_state.start_number() + factory_state.block_no(), - _ => unreachable!("Mode not covered!"), - }; - let to = RA::gen_random_account_id(&seed); - - let amount = RA::minimum_balance(); - - let transfer = factory_state.transfer_extrinsic( - &from.0, - &from.1, - &to, - &amount, - version, - &genesis_hash, - &prior_block_hash, - ); - - let inherents = RA::inherent_extrinsics(&factory_state); - let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents) - .expect("Failed to create inherent extrinsics"); - - let block = create_block::(&client, transfer, inherents); - - factory_state.set_block_no(factory_state.block_no() + RA::Number::one()); - - info!( - "Created block {} with hash {}. Transferring {} from {} to {}.", - factory_state.block_no(), - prior_block_hash, - amount, - from.0, - to - ); - - Some(block) -} diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 8f122a35a02e67a63acdd3bf7f551d779f668d9d..328bc5e064a22de21ac4ecedfc6b4e565183601e 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -1,15 +1,17 @@ [package] name = "chain-spec-builder" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] ansi_term = "0.12.1" -sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } -node-cli = { version = "2.0.0", path = "../../node/cli" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../../client/keystore" } +node-cli = { version = "2.0.0-alpha.1", path = "../../node/cli" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } rand = "0.7.2" structopt = "0.3.8" diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 9b7db6699ffe21f73ace5ab77bfc1245ae130d28..49a57d7de02ba019fc042631655fb04ca7a75464 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -1,33 +1,37 @@ [package] name = "subkey" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] futures = "0.1.29" -sp-core = { version = "*", path = "../../../primitives/core" } -node-runtime = { version = "*", path = "../../node/runtime" } -node-primitives = { version = "*", path = "../../node/primitives" } -sp-runtime = { version = "*", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +node-runtime = { version = "2.0.0-alpha.1", path = "../../node/runtime" } +node-primitives = { version = "2.0.0-alpha.1", path = "../../node/primitives" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } rand = "0.7.2" clap = "2.33.0" -tiny-bip39 = "0.6.2" +tiny-bip39 = "0.7" rustc-hex = "2.0.1" substrate-bip39 = "0.3.1" hex = "0.4.0" hex-literal = "0.2.1" codec = { package = "parity-scale-codec", version = "1.0.0" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } +frame-system = { version = "2.0.0-alpha.1", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", path = "../../../frame/transaction-payment" } rpassword = "4.0.1" itertools = "0.8.2" derive_more = { version = "0.99.2" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } +sc-rpc = { version = "2.0.0-alpha.1", path = "../../../client/rpc" } jsonrpc-core-client = { version = "14.0.3", features = ["http"] } hyper = "0.12.35" +libp2p = "0.16.1" +serde_json = "1.0" [features] bench = [] diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index c2fd7e28c254861345d42de49993431358740d47..33209692caffb87284318bc59962ab56bcb6cbf7 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -23,8 +23,10 @@ use clap::{App, ArgMatches, SubCommand}; use codec::{Decode, Encode}; use hex_literal::hex; use itertools::Itertools; +use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey}; use node_primitives::{Balance, Hash, Index, AccountId, Signature}; use node_runtime::{BalancesCall, Call, Runtime, SignedPayload, UncheckedExtrinsic, VERSION}; +use serde_json::json; use sp_core::{ crypto::{set_default_ss58_version, Ss58AddressFormat, Ss58Codec}, ed25519, sr25519, ecdsa, Pair, Public, H256, hexdisplay::HexDisplay, @@ -37,6 +39,24 @@ use std::{ mod rpc; mod vanity; +enum OutputType { + Json, + Text, +} + +impl<'a> TryFrom<&'a str> for OutputType { + type Error = (); + + fn try_from(s: &'a str) -> Result { + match s { + "json" => Ok(OutputType::Json), + "text" => Ok(OutputType::Text), + _ => Err(()), + } + } + +} + trait Crypto: Sized { type Pair: Pair; type Public: Public + Ss58Codec + AsRef<[u8]> + std::hash::Hash; @@ -55,50 +75,97 @@ trait Crypto: Sized { uri: &str, password: Option<&str>, network_override: Option, + output: OutputType, ) where ::Public: PublicT, { if let Ok((pair, seed)) = Self::Pair::from_phrase(uri, password) { let public_key = Self::public_from_pair(&pair); - println!("Secret phrase `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - format_seed::(seed), - format_public_key::(public_key.clone()), - format_account_id::(public_key), - Self::ss58_from_pair(&pair) - ); + + match output { + OutputType::Json => { + let json = json!({ + "secretPhrase": uri, + "secretSeed": format_seed::(seed), + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key), + "ss58Address": Self::ss58_from_pair(&pair), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Secret phrase `{}` is account:\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + format_seed::(seed), + format_public_key::(public_key.clone()), + format_account_id::(public_key), + Self::ss58_from_pair(&pair), + ); + }, + } } else if let Ok((pair, seed)) = Self::Pair::from_string_with_seed(uri, password) { let public_key = Self::public_from_pair(&pair); - println!("Secret Key URI `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, - format_public_key::(public_key.clone()), - format_account_id::(public_key), - Self::ss58_from_pair(&pair) - ); + + match output { + OutputType::Json => { + let json = json!({ + "secretKeyUri": uri, + "secretSeed": if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key), + "ss58Address": Self::ss58_from_pair(&pair), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Secret Key URI `{}` is account:\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, + format_public_key::(public_key.clone()), + format_account_id::(public_key), + Self::ss58_from_pair(&pair), + ); + }, + } + } else if let Ok((public_key, v)) = ::Public::from_string_with_version(uri) { let v = network_override.unwrap_or(v); - println!("Public Key URI `{}` is account:\n \ - Network ID/version: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - String::from(v), - format_public_key::(public_key.clone()), - format_account_id::(public_key.clone()), - public_key.to_ss58check_with_version(v) - ); + + match output { + OutputType::Json => { + let json = json!({ + "publicKeyUri": uri, + "networkId": String::from(v), + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key.clone()), + "ss58Address": public_key.to_ss58check_with_version(v), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Public Key URI `{}` is account:\n \ + Network ID/version: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + String::from(v), + format_public_key::(public_key.clone()), + format_account_id::(public_key.clone()), + public_key.to_ss58check_with_version(v), + ); + }, + } } else { println!("Invalid phrase/URI given"); } @@ -165,6 +232,7 @@ fn get_usage() -> String { [network] -n, --network 'Specify a network. One of {}. Default is {}' [password] -p, --password 'The password for the key' --password-interactive 'You will be prompted for the password for the key.' + [output] -o, --output 'Specify an output format. One of text, json. Default is text.' ", networks, default_network) } @@ -181,6 +249,9 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { 'The number of words in the phrase to generate. One of 12 \ (default), 15, 18, 21 and 24.' "), + SubCommand::with_name("generate-node-key") + .about("Generate a random node libp2p key, save it to file and print its peer ID") + .args_from_usage("[file] 'Name of file to save secret key to'"), SubCommand::with_name("inspect") .about("Gets a public key and a SS58 address from the provided Secret URI") .args_from_usage("[uri] 'A Key URI to be inspected. May be a secret seed, \ @@ -209,7 +280,7 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { SubCommand::with_name("transfer") .about("Author and sign a Node pallet_balances::Transfer transaction with a given (secret) key") .args_from_usage(" - -g, --genesis 'The genesis hash or a recognised \ + -g, --genesis 'The genesis hash or a recognized \ chain identifier (dev, elm, alex).' 'The signing secret key URI.' 'The destination account public key URI.' @@ -329,13 +400,31 @@ where if let Some(network) = maybe_network { set_default_ss58_version(network); } + + let output: OutputType = match matches.value_of("output").map(TryInto::try_into) { + Some(Err(_)) => return Err(Error::Static("Invalid output name. See --help for available outputs.")), + Some(Ok(v)) => v, + None => OutputType::Text, + }; + match matches.subcommand() { ("generate", Some(matches)) => { let mnemonic = generate_mnemonic(matches)?; - C::print_from_uri(mnemonic.phrase(), password, maybe_network); + C::print_from_uri(mnemonic.phrase(), password, maybe_network, output); + } + ("generate-node-key", Some(matches)) => { + let file = matches.value_of("file").ok_or(Error::Static("Output file name is required"))?; + + let keypair = libp2p_ed25519::Keypair::generate(); + let secret = keypair.secret(); + let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id(); + + fs::write(file, secret.as_ref())?; + + println!("{}", peer_id); } ("inspect", Some(matches)) => { - C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network); + C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network, output); } ("sign", Some(matches)) => { let suri = get_uri("suri", &matches)?; @@ -364,7 +453,7 @@ where .unwrap_or_default(); let result = vanity::generate_key::(&desired)?; let formated_seed = format_seed::(result.seed); - C::print_from_uri(&formated_seed, None, maybe_network); + C::print_from_uri(&formated_seed, None, maybe_network, output); } ("transfer", Some(matches)) => { let signer = read_pair::(matches.value_of("from"), password)?; @@ -483,7 +572,7 @@ fn read_genesis_hash(matches: &ArgMatches) -> Result { "elm" => hex!["10c08714a10c7da78f40a60f6f732cf0dba97acfb5e2035445b032386157d5c3"].into(), "alex" => hex!["dcd1346701ca8396496e52aa2785b1748deb6db09551b72159dcb3e08991025b"].into(), h => Decode::decode(&mut &decode_hex(h)?[..]) - .expect("Invalid genesis hash or unrecognised chain identifier"), + .expect("Invalid genesis hash or unrecognized chain identifier"), }; println!( "Using a genesis hash of {}", diff --git a/client/Cargo.toml b/client/Cargo.toml index c89fe88145d148a6d64e70dcae4adac7eb97021d..0ae03b56ef4ff772955c407ebd86d2bd74581cdc 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -1,41 +1,43 @@ [package] name = "sc-client" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-block-builder = { version = "0.8", path = "block-builder" } -sc-client-api = { version = "2.0.0", path = "api" } +sc-block-builder = { version = "0.8.0-alpha.1", path = "block-builder" } +sc-client-api = { version = "2.0.0-alpha.1", path = "api" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus = { version = "0.8", path = "../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8", path = "executor" } -sp-externalities = { version = "0.8.0", path = "../primitives/externalities" } +sc-executor = { version = "0.8.0-alpha.1", path = "executor" } +sp-externalities = { version = "0.8.0-alpha.1", path = "../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1", features = ["compat"] } hash-db = { version = "0.15.2" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0", path = "../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../primitives/keyring" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../primitives/keyring" } kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../primitives/core" } -sp-std = { version = "2.0.0", path = "../primitives/std" } -sp-version = { version = "2.0.0", path = "../primitives/version" } -sp-api = { version = "2.0.0", path = "../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../primitives/blockchain" } -sp-state-machine = { version = "0.8", path = "../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "telemetry" } -sp-trie = { version = "2.0.0", path = "../primitives/trie" } +sp-core = { version = "2.0.0-alpha.1", path = "../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", path = "../primitives/std" } +sp-version = { version = "2.0.0-alpha.1", path = "../primitives/version" } +sp-api = { version = "2.0.0-alpha.1", path = "../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../primitives/runtime" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../primitives/blockchain" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "telemetry" } +sp-trie = { version = "2.0.0-alpha.1", path = "../primitives/trie" } tracing = "0.1.10" [dev-dependencies] env_logger = "0.7.0" tempfile = "3.1.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../test-utils/runtime/client" } kvdb-memorydb = "0.4.0" -sp-panic-handler = { version = "2.0.0", path = "../primitives/panic-handler" } +sp-panic-handler = { version = "2.0.0-alpha.1", path = "../primitives/panic-handler" } diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 27a40c4d94cfbd70a77bfeac94dc770c264b7268..6256a99bb88acd3a2a34e3460629ba812d3ac8e0 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -1,35 +1,37 @@ [package] name = "sc-client-api" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8", path = "../executor" } -sp-externalities = { version = "0.8.0", path = "../../primitives/externalities" } +sc-executor = { version = "0.8.0-alpha.1", path = "../executor" } +sp-externalities = { version = "0.8.0-alpha.1", path = "../../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1" } hash-db = { version = "0.15.2", default-features = false } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } +sp-inherents = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-version = { version = "2.0.0", default-features = false, path = "../../primitives/version" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-version = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/version" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } +sp-trie = { version = "2.0.0-alpha.1", path = "../../primitives/trie" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } [dev-dependencies] -sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } +sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 65952ce787bb23552a10989faa18dc64b72e5982..7503ce4a79e4627eb9fac7527e3670cfb35c23f3 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -125,6 +125,8 @@ pub struct IoInfo { pub state_reads: u64, /// State reads (keys) from cache. pub state_reads_cache: u64, + /// State reads (keys) from cache. + pub state_writes: u64, } /// Usage statistics for running client instance. @@ -143,7 +145,7 @@ pub struct UsageInfo { impl fmt::Display for UsageInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, - "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total)", + "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)", self.memory.state_cache, self.memory.database_cache, self.io.transactions, @@ -152,6 +154,7 @@ impl fmt::Display for UsageInfo { self.io.average_transaction_size, self.io.state_reads_cache, self.io.state_reads, + self.io.state_writes, ) } } diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 1a986a23a47dc7a2cf97676e0c39d63fb18ecc5f..10d33c20e679c6eeddb4368da3c18d429d2ade6b 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -64,7 +64,7 @@ impl Default for ExecutionStrategies { /// Generate the starting set of ExternalitiesExtensions based upon the given capabilities pub trait ExtensionsFactory: Send + Sync { - /// Make `Extensions` for given Capapbilities + /// Make `Extensions` for given `Capabilities`. fn extensions_for(&self, capabilities: offchain::Capabilities) -> Extensions; } @@ -77,7 +77,7 @@ impl ExtensionsFactory for () { /// A producer of execution extensions for offchain calls. /// /// This crate aggregates extensions available for the offchain calls -/// and is responsbile to produce a right `Extensions` object +/// and is responsible for producing a correct `Extensions` object. /// for each call, based on required `Capabilities`. pub struct ExecutionExtensions { strategies: ExecutionStrategies, @@ -125,7 +125,7 @@ impl ExecutionExtensions { /// To break retain cycle between `Client` and `TransactionPool` we require this /// extension to be a `Weak` reference. /// That's also the reason why it's being registered lazily instead of - /// during initialisation. + /// during initialization. pub fn register_transaction_pool(&self, pool: Weak>) { *self.transaction_pool.write() = Some(pool); } diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 68144625b3f1591a7d509631e241b22d1fcb43c1..95e0d78240e030368dfdee9d79717c0f25f900e0 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -1,36 +1,38 @@ [package] name = "sc-authority-discovery" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [build-dependencies] prost-build = "0.6.1" [dependencies] -bytes = "0.4.12" +bytes = "0.5.0" codec = { package = "parity-scale-codec", default-features = false, version = "1.0.3" } derive_more = "0.99.2" futures = "0.3.1" -futures-timer = "2.0" -libp2p = { version = "0.15.0", default-features = false, features = ["secp256k1", "libp2p-websocket"] } +futures-timer = "3.0.1" +libp2p = { version = "0.16.1", default-features = false, features = ["secp256k1", "libp2p-websocket"] } log = "0.4.8" prost = "0.6.1" rand = "0.7.2" -sc-client-api = { version = "2.0.0", path = "../api" } -sc-keystore = { version = "2.0.0", path = "../keystore" } -sc-network = { version = "0.8", path = "../network" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../keystore" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } serde_json = "1.0.41" -sp-authority-discovery = { version = "2.0.0", path = "../../primitives/authority-discovery" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } +sp-authority-discovery = { version = "2.0.0-alpha.1", path = "../../primitives/authority-discovery" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } [dev-dependencies] env_logger = "0.7.0" quickcheck = "0.9.0" -sc-peerset = { version = "2.0.0", path = "../peerset" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client"} +sc-peerset = { version = "2.0.0-alpha.1", path = "../peerset" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client"} diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index 6260ac9a85b12ec0af93227048e8279438161c11..92dcc264502f03612463b44d128148a2aeddebe3 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -60,7 +60,6 @@ use libp2p::Multiaddr; use log::{debug, error, log_enabled, warn}; use prost::Message; use sc_client_api::blockchain::HeaderBackend; -use sc_network::specialization::NetworkSpecialization; use sc_network::{DhtEvent, ExHashT, NetworkStateInfo}; use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; use sp_core::crypto::{key_types, Pair}; @@ -477,10 +476,9 @@ pub trait NetworkProvider: NetworkStateInfo { fn get_value(&self, key: &libp2p::kad::record::Key); } -impl NetworkProvider for sc_network::NetworkService +impl NetworkProvider for sc_network::NetworkService where B: BlockT + 'static, - S: NetworkSpecialization, H: ExHashT, { fn set_priority_group( diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 6a013f7a74fce889d329bc930da20c8aca02cbd9..971279297ac3a4b20687729a31adf7a8e7dddbb2 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -1,28 +1,30 @@ [package] name = "sc-basic-authorship" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] log = "0.4.8" futures = "0.3.1" codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-client = { version = "0.8", path = "../" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../primitives/inherents" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-block-builder = { version = "0.8", path = "../block-builder" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../primitives/inherents" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } +sc-block-builder = { version = "0.8.0-alpha.1", path = "../block-builder" } tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] } [dev-dependencies] -sc-transaction-pool = { version = "2.0.0", path = "../../client/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../../client/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } parking_lot = "0.10.0" diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index a9e3d28e21e30614c8f880c493a07ea8f58606a0..a99453544e5f63e3a6a6e5d62845b465d4bc86d8 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -195,7 +195,7 @@ impl ProposerInner, inherent_data )? { - block_builder.push(extrinsic)?; + block_builder.push_trusted(extrinsic)?; } // proceed with transactions @@ -218,7 +218,7 @@ impl ProposerInner, let pending_tx_data = pending_tx.data().clone(); let pending_tx_hash = pending_tx.hash().clone(); trace!("[{:?}] Pushing to the block.", pending_tx_hash); - match sc_block_builder::BlockBuilder::push(&mut block_builder, pending_tx_data) { + match sc_block_builder::BlockBuilder::push_trusted(&mut block_builder, pending_tx_data) { Ok(()) => { debug!("[{:?}] Pushed to the block.", pending_tx_hash); } @@ -308,7 +308,7 @@ mod tests { // given let client = Arc::new(substrate_test_runtime_client::new()); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))) + BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 ); futures::executor::block_on( @@ -350,7 +350,7 @@ mod tests { .build_with_backend(); let client = Arc::new(client); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))) + BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 ); let genesis_hash = client.info().best_hash; let block_id = BlockId::Hash(genesis_hash); diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index 9f2bc6a761eb01f14d5a4b11a64989bdbd4b1114..e9087c89e07ed373ec2cb932b37b2b6822f570e3 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -26,7 +26,7 @@ //! # use substrate_test_runtime_client::{self, runtime::{Extrinsic, Transfer}, AccountKeyring}; //! # use sc_transaction_pool::{BasicPool, FullChainApi}; //! # let client = Arc::new(substrate_test_runtime_client::new()); -//! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone())))); +//! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0); //! // The first step is to create a `ProposerFactory`. //! let mut proposer_factory = ProposerFactory { //! client: client.clone(), diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 383a931b2f8998288a362713a6a24aacaeff70c9..94b7640d2f5750a419227c698815a39d13c639d4 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -1,17 +1,19 @@ [package] name = "sc-block-builder" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-consensus = { version = "0.8.0", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-block-builder = { version = "2.0.0", path = "../../primitives/block-builder" } -sc-client-api = { version = "2.0.0", path = "../api" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../primitives/block-builder" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } codec = { package = "parity-scale-codec", version = "1.0.6", features = ["derive"] } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index d0eb8b2892630f177e6cce45e9def226643b077a..a93ad137835edfe3f91221da0647aa30a3bba6b2 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -124,48 +124,52 @@ where /// Push onto the block's list of extrinsics. /// - /// This will ensure the extrinsic can be validly executed (by executing it); + /// This will ensure the extrinsic can be validly executed (by executing it). pub fn push(&mut self, xt: ::Extrinsic) -> Result<(), ApiErrorFor> { + self.push_internal(xt, false) + } + + /// Push onto the block's list of extrinsics. + /// + /// This will treat incoming extrinsic `xt` as trusted and skip signature check (for signed transactions). + pub fn push_trusted(&mut self, xt: ::Extrinsic) -> Result<(), ApiErrorFor> { + self.push_internal(xt, true) + } + + fn push_internal(&mut self, xt: ::Extrinsic, skip_signature: bool) -> Result<(), ApiErrorFor> { let block_id = &self.block_id; let extrinsics = &mut self.extrinsics; - if self + let use_trusted = skip_signature && self .api .has_api_with::>, _>( block_id, - |version| version < 4, - )? - { - // Run compatibility fallback for v3. - self.api.map_api_result(|api| { - #[allow(deprecated)] - match api.apply_extrinsic_before_version_4_with_context( + |version| version >= 5, + )?; + + self.api.map_api_result(|api| { + let apply_result = if use_trusted { + api.apply_trusted_extrinsic_with_context( block_id, ExecutionContext::BlockConstruction, xt.clone(), - )? { - Ok(_) => { - extrinsics.push(xt); - Ok(()) - } - Err(e) => Err(ApplyExtrinsicFailed::from(e).into()), - } - }) - } else { - self.api.map_api_result(|api| { - match api.apply_extrinsic_with_context( + )? + } else { + api.apply_extrinsic_with_context( block_id, ExecutionContext::BlockConstruction, xt.clone(), - )? { - Ok(_) => { - extrinsics.push(xt); - Ok(()) - } - Err(tx_validity) => Err(ApplyExtrinsicFailed::Validity(tx_validity).into()), + )? + }; + + match apply_result { + Ok(_) => { + extrinsics.push(xt); + Ok(()) } - }) - } + Err(tx_validity) => Err(ApplyExtrinsicFailed::Validity(tx_validity).into()), + } + }) } /// Consume the builder to build a valid `Block` containing all pushed extrinsics. diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 222914145e160cbc71e9e7007a7e1f70c01d1380..e1fc4fbc20c8d06eebec21472e3061c5688078e3 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "sc-chain-spec" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-chain-spec-derive = { version = "2.0.0", path = "./derive" } +sc-chain-spec-derive = { version = "2.0.0-alpha.1", path = "./derive" } impl-trait-for-tuples = "0.1.3" -sc-network = { version = "0.8", path = "../network" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index 566948883bccb18890de62e54ae3a3547737357c..61fe7e9df0eed57019e55db9d99d6ecc463e5e8b 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-chain-spec-derive" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [lib] proc-macro = true diff --git a/client/chain-spec/derive/src/impls.rs b/client/chain-spec/derive/src/impls.rs index 2caf1c5a9c3ebbe148a5567e40695e95b66293dd..43874241f600f6ffdac987c954fdea7573f01d82 100644 --- a/client/chain-spec/derive/src/impls.rs +++ b/client/chain-spec/derive/src/impls.rs @@ -108,7 +108,7 @@ pub fn derive( let err = || { let err = Error::new( Span::call_site(), - "ChainSpecGroup is only avaible for structs with named fields." + "ChainSpecGroup is only available for structs with named fields." ).to_compile_error(); quote!( #err ).into() }; diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 81cbce5ea731c095f482293f6d19ca39e12a2698..a7e5738fc4bd8883b70697640f09111e5910901e 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -20,7 +20,7 @@ use std::borrow::Cow; use std::collections::HashMap; use std::fs::File; use std::path::PathBuf; -use std::rc::Rc; +use std::sync::Arc; use serde::{Serialize, Deserialize}; use sp_core::storage::{StorageKey, StorageData, ChildInfo, Storage, StorageChild}; use sp_runtime::BuildStorage; @@ -32,7 +32,7 @@ use sc_telemetry::TelemetryEndpoints; enum GenesisSource { File(PathBuf), Binary(Cow<'static, [u8]>), - Factory(Rc G>), + Factory(Arc G + Send + Sync>), } impl Clone for GenesisSource { @@ -117,8 +117,8 @@ struct ChildRawStorage { #[serde(deny_unknown_fields)] /// Storage content for genesis block. struct RawGenesis { - pub top: GenesisStorage, - pub children: HashMap, + top: GenesisStorage, + children: HashMap, } #[derive(Serialize, Deserialize)] @@ -134,14 +134,14 @@ enum Genesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] struct ClientSpec { - pub name: String, - pub id: String, - pub boot_nodes: Vec, - pub telemetry_endpoints: Option, - pub protocol_id: Option, - pub properties: Option, + name: String, + id: String, + boot_nodes: Vec, + telemetry_endpoints: Option, + protocol_id: Option, + properties: Option, #[serde(flatten)] - pub extensions: E, + extensions: E, // Never used, left only for backward compatibility. consensus_engine: (), #[serde(skip_serializing)] @@ -215,7 +215,7 @@ impl ChainSpec { } /// Create hardcoded spec. - pub fn from_genesis G + 'static>( + pub fn from_genesis G + 'static + Send + Sync>( name: &str, id: &str, constructor: F, @@ -239,7 +239,7 @@ impl ChainSpec { ChainSpec { client_spec, - genesis: GenesisSource::Factory(Rc::new(constructor)), + genesis: GenesisSource::Factory(Arc::new(constructor)), } } } @@ -336,7 +336,7 @@ mod tests { type TestSpec = ChainSpec; #[test] - fn should_deserailize_example_chain_spec() { + fn should_deserialize_example_chain_spec() { let spec1 = TestSpec::from_json_bytes(Cow::Owned( include_bytes!("../res/chain_spec.json").to_vec() )).unwrap(); diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index e302d53d55a61e00c05f25ad4df2095ba41f8817..9c724296389c2e75be5f2a3c2a9801438a63601c 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "sc-cli" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Substrate CLI interface." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] clap = "2.33.0" @@ -21,19 +23,21 @@ tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } futures = "0.3.1" fdlimit = "0.1.1" serde_json = "1.0.41" -sp-panic-handler = { version = "2.0.0", path = "../../primitives/panic-handler" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-network = { version = "0.8", path = "../network" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sc-service = { version = "0.8", default-features = false, path = "../service" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } +sc-informant = { version = "0.8.0-alpha.1", path = "../informant" } +sp-panic-handler = { version = "2.0.0-alpha.1", path = "../../primitives/panic-handler" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sc-service = { version = "0.8.0-alpha.1", default-features = false, path = "../service" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } +prometheus-exporter = { path = "../../utils/prometheus" , version = "0.8.0-alpha.1"} +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } names = "0.11.0" structopt = "0.3.8" -sc-tracing = { version = "2.0.0", path = "../tracing" } +sc-tracing = { version = "2.0.0-alpha.1", path = "../tracing" } chrono = "0.4.10" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } diff --git a/client/cli/src/execution_strategy.rs b/client/cli/src/arg_enums.rs similarity index 51% rename from client/cli/src/execution_strategy.rs rename to client/cli/src/arg_enums.rs index 888d7b6c4a09680103c15b44e27b67b700f6505c..384087bec0dbe4538c7227e36a6dbce7d789586b 100644 --- a/client/cli/src/execution_strategy.rs +++ b/client/cli/src/arg_enums.rs @@ -14,10 +14,74 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +// NOTE: we allow missing docs here because arg_enum! creates the function variants without doc #![allow(missing_docs)] use structopt::clap::arg_enum; +arg_enum! { + /// How to execute Wasm runtime code + #[allow(missing_docs)] + #[derive(Debug, Clone, Copy)] + pub enum WasmExecutionMethod { + // Uses an interpreter. + Interpreted, + // Uses a compiled runtime. + Compiled, + } +} + +impl WasmExecutionMethod { + /// Returns list of variants that are not disabled by feature flags. + pub fn enabled_variants() -> Vec<&'static str> { + Self::variants() + .iter() + .cloned() + .filter(|&name| cfg!(feature = "wasmtime") || name != "Compiled") + .collect() + } +} + +impl Into for WasmExecutionMethod { + fn into(self) -> sc_service::config::WasmExecutionMethod { + match self { + WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, + #[cfg(feature = "wasmtime")] + WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, + #[cfg(not(feature = "wasmtime"))] + WasmExecutionMethod::Compiled => panic!( + "Substrate must be compiled with \"wasmtime\" feature for compiled Wasm execution" + ), + } + } +} + +arg_enum! { + #[allow(missing_docs)] + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum TracingReceiver { + Log, + Telemetry, + } +} + +impl Into for TracingReceiver { + fn into(self) -> sc_tracing::TracingReceiver { + match self { + TracingReceiver::Log => sc_tracing::TracingReceiver::Log, + TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry, + } + } +} + +arg_enum! { + #[allow(missing_docs)] + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum NodeKeyType { + Ed25519 + } +} + arg_enum! { /// How to execute blocks #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -33,6 +97,17 @@ arg_enum! { } } +impl Into for ExecutionStrategy { + fn into(self) -> sc_client_api::ExecutionStrategy { + match self { + ExecutionStrategy::Native => sc_client_api::ExecutionStrategy::NativeWhenPossible, + ExecutionStrategy::Wasm => sc_client_api::ExecutionStrategy::AlwaysWasm, + ExecutionStrategy::Both => sc_client_api::ExecutionStrategy::Both, + ExecutionStrategy::NativeElseWasm => sc_client_api::ExecutionStrategy::NativeElseWasm, + } + } +} + impl ExecutionStrategy { /// Returns the variant as `'&static str`. pub fn as_str(&self) -> &'static str { diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b71207efabf8de381d7066b284356e6c2663c11 --- /dev/null +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -0,0 +1,104 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::{Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; +use crate::params::NodeKeyParams; + +/// The `build-spec` command used to build a specification. +#[derive(Debug, StructOpt, Clone)] +pub struct BuildSpecCmd { + /// Force raw genesis storage output. + #[structopt(long = "raw")] + pub raw: bool, + + /// Disable adding the default bootnode to the specification. + /// + /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the + /// specification when no bootnode exists. + #[structopt(long = "disable-default-bootnode")] + pub disable_default_bootnode: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub node_key_params: NodeKeyParams, +} + +impl BuildSpecCmd { + /// Run the build-spec command + pub fn run( + self, + config: Configuration, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + { + info!("Building chain spec"); + let mut spec = config.expect_chain_spec().clone(); + let raw_output = self.raw; + + if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { + let keys = config.network.node_key.into_keypair()?; + let peer_id = keys.public().into_peer_id(); + let addr = build_multiaddr![ + Ip4([127, 0, 0, 1]), + Tcp(30333u16), + P2p(peer_id) + ]; + spec.add_boot_node(addr) + } + + let json = sc_service::chain_ops::build_spec(spec, raw_output)?; + + print!("{}", json); + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + + let net_config_path = config + .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) + .expect("We provided a base_path"); + + self.node_key_params.update_config(&mut config, Some(&net_config_path))?; + + Ok(()) + } +} + diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..1036be16de4ebcde466710781f4cfecb283216eb --- /dev/null +++ b/client/cli/src/commands/check_block_cmd.rs @@ -0,0 +1,105 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::str::FromStr; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, Roles, ChainSpec, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_runtime::generic::BlockId; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::SharedParams; +use crate::params::ImportParams; + +/// The `check-block` command used to validate blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct CheckBlockCmd { + /// Block hash or number + #[structopt(value_name = "HASH or NUMBER")] + pub input: String, + + /// The default number of 64KB pages to ever allocate for Wasm execution. + /// + /// Don't alter this unless you know what you're doing. + #[structopt(long = "default-heap-pages", value_name = "COUNT")] + pub default_heap_pages: Option, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, +} + +impl CheckBlockCmd { + /// Run the check-block command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; + let block_id = match FromStr::from_str(input) { + Ok(hash) => BlockId::hash(hash), + Err(_) => match self.input.parse::() { + Ok(n) => BlockId::number((n as u32).into()), + Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), + } + }; + + let start = std::time::Instant::now(); + run_until_exit(config, |config| { + Ok(builder(config)?.check_block(block_id)) + })?; + println!("Completed in {} ms.", start.elapsed().as_millis()); + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..cdfa463c6d561c1445bf1dbcf42aa9c00f3f6259 --- /dev/null +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -0,0 +1,117 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::io; +use std::fs; +use std::path::PathBuf; +use std::fmt::Debug; +use log::info; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, + config::DatabaseConfig, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::{SharedParams, BlockNumber, PruningParams}; + +/// The `export-blocks` command used to export blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct ExportBlocksCmd { + /// Output file name or stdout if unspecified. + #[structopt(parse(from_os_str))] + pub output: Option, + + /// Specify starting block number. + /// + /// Default is 1. + #[structopt(long = "from", value_name = "BLOCK")] + pub from: Option, + + /// Specify last block number. + /// + /// Default is best block. + #[structopt(long = "to", value_name = "BLOCK")] + pub to: Option, + + /// Use JSON output rather than binary. + #[structopt(long = "json")] + pub json: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, +} + +impl ExportBlocksCmd { + /// Run the export-blocks command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { + info!("DB path: {}", path.display()); + } + let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); + let to = self.to.as_ref().and_then(|t| t.parse().ok()); + + let json = self.json; + + let file: Box = match &self.output { + Some(filename) => Box::new(fs::File::create(filename)?), + None => Box::new(io::stdout()), + }; + + run_until_exit(config, |config| { + Ok(builder(config)?.export_blocks(file, from.into(), to, json)) + }) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.pruning_params.update_config(&mut config, Roles::FULL, true)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..60a57ab78d1b1b3563d4f3a047f3be70152820eb --- /dev/null +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -0,0 +1,107 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::io::{Read, Seek, self}; +use std::fs; +use std::path::PathBuf; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::SharedParams; +use crate::params::ImportParams; + +/// The `import-blocks` command used to import blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct ImportBlocksCmd { + /// Input file or stdin if unspecified. + #[structopt(parse(from_os_str))] + pub input: Option, + + /// The default number of 64KB pages to ever allocate for Wasm execution. + /// + /// Don't alter this unless you know what you're doing. + #[structopt(long = "default-heap-pages", value_name = "COUNT")] + pub default_heap_pages: Option, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, +} + +/// Internal trait used to cast to a dynamic type that implements Read and Seek. +trait ReadPlusSeek: Read + Seek {} + +impl ReadPlusSeek for T {} + +impl ImportBlocksCmd { + /// Run the import-blocks command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let file: Box = match &self.input { + Some(filename) => Box::new(fs::File::open(filename)?), + None => { + let mut buffer = Vec::new(); + io::stdin().read_to_end(&mut buffer)?; + Box::new(io::Cursor::new(buffer)) + }, + }; + + run_until_exit(config, |config| { + Ok(builder(config)?.import_blocks(file, false)) + }) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e9f991c7458837a97926e7b47520ba331bacc93d --- /dev/null +++ b/client/cli/src/commands/mod.rs @@ -0,0 +1,145 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +mod runcmd; +mod export_blocks_cmd; +mod build_spec_cmd; +mod import_blocks_cmd; +mod check_block_cmd; +mod revert_cmd; +mod purge_chain_cmd; + +use std::fmt::Debug; +use structopt::StructOpt; + +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; + +pub use crate::commands::runcmd::RunCmd; +pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; +pub use crate::commands::build_spec_cmd::BuildSpecCmd; +pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; +pub use crate::commands::check_block_cmd::CheckBlockCmd; +pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::purge_chain_cmd::PurgeChainCmd; + +/// default sub directory to store network config +const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; + +/// All core commands that are provided by default. +/// +/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From +/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of +/// `Run` are exported as main executable parameters. +#[derive(Debug, Clone, StructOpt)] +pub enum Subcommand { + /// Build a spec.json file, outputing to stdout. + BuildSpec(build_spec_cmd::BuildSpecCmd), + + /// Export blocks to a file. + ExportBlocks(export_blocks_cmd::ExportBlocksCmd), + + /// Import blocks from file. + ImportBlocks(import_blocks_cmd::ImportBlocksCmd), + + /// Validate a single block. + CheckBlock(check_block_cmd::CheckBlockCmd), + + /// Revert chain to the previous state. + Revert(revert_cmd::RevertCmd), + + /// Remove the whole chain data. + PurgeChain(purge_chain_cmd::PurgeChainCmd), +} + +impl Subcommand { + /// Get the shared parameters of a `CoreParams` command + pub fn get_shared_params(&self) -> &SharedParams { + use Subcommand::*; + + match self { + BuildSpec(params) => ¶ms.shared_params, + ExportBlocks(params) => ¶ms.shared_params, + ImportBlocks(params) => ¶ms.shared_params, + CheckBlock(params) => ¶ms.shared_params, + Revert(params) => ¶ms.shared_params, + PurgeChain(params) => ¶ms.shared_params, + } + } + + /// Run any `CoreParams` command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + match self { + Subcommand::BuildSpec(cmd) => cmd.run(config), + Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), + Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), + Subcommand::CheckBlock(cmd) => cmd.run(config, builder), + Subcommand::PurgeChain(cmd) => cmd.run(config), + Subcommand::Revert(cmd) => cmd.run(config, builder), + } + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + match self { + Subcommand::BuildSpec(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::ExportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::ImportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::CheckBlock(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::PurgeChain(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::Revert(cmd) => cmd.update_config(&mut config, spec_factory, version), + } + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + self.get_shared_params().init(version) + } +} diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..b7c559e5cc359eec154a250a4c41cb2b8f5bd70d --- /dev/null +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -0,0 +1,106 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::io::{Write, self}; +use std::fs; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec, + config::{DatabaseConfig}, +}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; + +/// The `purge-chain` command used to remove the whole chain. +#[derive(Debug, StructOpt, Clone)] +pub struct PurgeChainCmd { + /// Skip interactive prompt by answering yes automatically. + #[structopt(short = "y")] + pub yes: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, +} + +impl PurgeChainCmd { + /// Run the purge command + pub fn run( + self, + config: Configuration, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + { + let db_path = match config.expect_database() { + DatabaseConfig::Path { path, .. } => path, + _ => { + eprintln!("Cannot purge custom database implementation"); + return Ok(()); + } + }; + + if !self.yes { + print!("Are you sure to remove {:?}? [y/N]: ", &db_path); + io::stdout().flush().expect("failed to flush stdout"); + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + let input = input.trim(); + + match input.chars().nth(0) { + Some('y') | Some('Y') => {}, + _ => { + println!("Aborted"); + return Ok(()); + }, + } + } + + match fs::remove_dir_all(&db_path) { + Ok(_) => { + println!("{:?} removed.", &db_path); + Ok(()) + }, + Err(ref err) if err.kind() == io::ErrorKind::NotFound => { + eprintln!("{:?} did not exist.", &db_path); + Ok(()) + }, + Err(err) => Result::Err(err.into()) + } + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..f0c534898effd33ac33b979c322a98d78b3877cd --- /dev/null +++ b/client/cli/src/commands/revert_cmd.rs @@ -0,0 +1,83 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::params::{BlockNumber, SharedParams, PruningParams}; + +/// The `revert` command used revert the chain to a previous state. +#[derive(Debug, StructOpt, Clone)] +pub struct RevertCmd { + /// Number of blocks to revert. + #[structopt(default_value = "256")] + pub num: BlockNumber, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, +} + +impl RevertCmd { + /// Run the revert command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let blocks = self.num.parse()?; + builder(config)?.revert_chain(blocks)?; + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.pruning_params.update_config(&mut config, Roles::FULL, true)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/runcmd.rs b/client/cli/src/commands/runcmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..f29bc3c743b64e6156e98de674691e389772e106 --- /dev/null +++ b/client/cli/src/commands/runcmd.rs @@ -0,0 +1,737 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use std::net::SocketAddr; +use std::fs; +use log::info; +use structopt::{StructOpt, clap::arg_enum}; +use names::{Generator, Name}; +use regex::Regex; +use chrono::prelude::*; +use sc_service::{ + AbstractService, Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec, Roles, + config::KeystoreConfig, +}; +use sc_telemetry::TelemetryEndpoints; + +use crate::VersionInfo; +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::params::NetworkConfigurationParams; +use crate::params::TransactionPoolParams; +use crate::runtime::run_service_until_exit; + +/// The maximum number of characters for a node name. +const NODE_NAME_MAX_LENGTH: usize = 32; + +/// default sub directory for the key store +const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; + +arg_enum! { + /// Whether off-chain workers are enabled. + #[allow(missing_docs)] + #[derive(Debug, Clone)] + pub enum OffchainWorkerEnabled { + Always, + Never, + WhenValidating, + } +} + +/// The `run` command used to run a node. +#[derive(Debug, StructOpt, Clone)] +pub struct RunCmd { + /// Enable validator mode. + /// + /// The node will be started with the authority role and actively + /// participate in any consensus task that it can (e.g. depending on + /// availability of local keys). + #[structopt( + long = "validator", + conflicts_with_all = &[ "sentry" ] + )] + pub validator: bool, + + /// Enable sentry mode. + /// + /// The node will be started with the authority role and participate in + /// consensus tasks as an "observer", it will never actively participate + /// regardless of whether it could (e.g. keys are available locally). This + /// mode is useful as a secure proxy for validators (which would run + /// detached from the network), since we want this node to participate in + /// the full consensus protocols in order to have all needed consensus data + /// available to relay to private nodes. + #[structopt( + long = "sentry", + conflicts_with_all = &[ "validator", "light" ] + )] + pub sentry: bool, + + /// Disable GRANDPA voter when running in validator mode, otherwise disables the GRANDPA observer. + #[structopt(long = "no-grandpa")] + pub no_grandpa: bool, + + /// Experimental: Run in light client mode. + #[structopt(long = "light", conflicts_with = "sentry")] + pub light: bool, + + /// Listen to all RPC interfaces. + /// + /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy + /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. + /// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks. + #[structopt(long = "rpc-external")] + pub rpc_external: bool, + + /// Listen to all RPC interfaces. + /// + /// Same as `--rpc-external`. + #[structopt(long = "unsafe-rpc-external")] + pub unsafe_rpc_external: bool, + + /// Listen to all Websocket interfaces. + /// + /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy + /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. + /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. + #[structopt(long = "ws-external")] + pub ws_external: bool, + + /// Listen to all Websocket interfaces. + /// + /// Same as `--ws-external` but doesn't warn you about it. + #[structopt(long = "unsafe-ws-external")] + pub unsafe_ws_external: bool, + + /// Listen to all Prometheus data source interfaces. + /// + /// Default is local. + #[structopt(long = "prometheus-external")] + pub prometheus_external: bool, + + /// Specify HTTP RPC server TCP port. + #[structopt(long = "rpc-port", value_name = "PORT")] + pub rpc_port: Option, + + /// Specify WebSockets RPC server TCP port. + #[structopt(long = "ws-port", value_name = "PORT")] + pub ws_port: Option, + + /// Maximum number of WS RPC server connections. + #[structopt(long = "ws-max-connections", value_name = "COUNT")] + pub ws_max_connections: Option, + + /// Specify browser Origins allowed to access the HTTP & WS RPC servers. + /// + /// A comma-separated list of origins (protocol://domain or special `null` + /// value). Value of `all` will disable origin validation. Default is to + /// allow localhost, https://polkadot.js.org and + /// https://substrate-ui.parity.io origins. When running in --dev mode the + /// default is to allow all origins. + #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] + pub rpc_cors: Option, + + /// Specify Prometheus data source server TCP Port. + #[structopt(long = "prometheus-port", value_name = "PORT")] + pub prometheus_port: Option, + + /// Do not expose a Prometheus metric endpoint. + /// + /// Prometheus metric endpoint is enabled by default. + #[structopt(long = "no-prometheus")] + pub no_prometheus: bool, + + /// The human-readable name for this node. + /// + /// The node name will be reported to the telemetry server, if enabled. + #[structopt(long = "name", value_name = "NAME")] + pub name: Option, + + /// Disable connecting to the Substrate telemetry server. + /// + /// Telemetry is on by default on global chains. + #[structopt(long = "no-telemetry")] + pub no_telemetry: bool, + + /// The URL of the telemetry server to connect to. + /// + /// This flag can be passed multiple times as a mean to specify multiple + /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting + /// the least verbosity. If no verbosity level is specified the default is + /// 0. + #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] + pub telemetry_endpoints: Vec<(String, u8)>, + + /// Should execute offchain workers on every block. + /// + /// By default it's only enabled for nodes that are authoring new blocks. + #[structopt( + long = "offchain-worker", + value_name = "ENABLED", + possible_values = &OffchainWorkerEnabled::variants(), + case_insensitive = true, + default_value = "WhenValidating" + )] + pub offchain_worker: OffchainWorkerEnabled, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub network_config: NetworkConfigurationParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pool_config: TransactionPoolParams, + + /// Shortcut for `--name Alice --validator` with session keys for `Alice` added to keystore. + #[structopt(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] + pub alice: bool, + + /// Shortcut for `--name Bob --validator` with session keys for `Bob` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] + pub bob: bool, + + /// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] + pub charlie: bool, + + /// Shortcut for `--name Dave --validator` with session keys for `Dave` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] + pub dave: bool, + + /// Shortcut for `--name Eve --validator` with session keys for `Eve` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] + pub eve: bool, + + /// Shortcut for `--name Ferdie --validator` with session keys for `Ferdie` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] + pub ferdie: bool, + + /// Shortcut for `--name One --validator` with session keys for `One` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] + pub one: bool, + + /// Shortcut for `--name Two --validator` with session keys for `Two` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] + pub two: bool, + + /// Enable authoring even when offline. + #[structopt(long = "force-authoring")] + pub force_authoring: bool, + + /// Specify custom keystore path. + #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] + pub keystore_path: Option, + + /// Use interactive shell for entering the password used by the keystore. + #[structopt( + long = "password-interactive", + conflicts_with_all = &[ "password", "password-filename" ] + )] + pub password_interactive: bool, + + /// Password used by the keystore. + #[structopt( + long = "password", + conflicts_with_all = &[ "password-interactive", "password-filename" ] + )] + pub password: Option, + + /// File that contains the password used by the keystore. + #[structopt( + long = "password-filename", + value_name = "PATH", + parse(from_os_str), + conflicts_with_all = &[ "password-interactive", "password" ] + )] + pub password_filename: Option +} + +impl RunCmd { + /// Get the `Sr25519Keyring` matching one of the flag + pub fn get_keyring(&self) -> Option { + use sp_keyring::Sr25519Keyring::*; + + if self.alice { Some(Alice) } + else if self.bob { Some(Bob) } + else if self.charlie { Some(Charlie) } + else if self.dave { Some(Dave) } + else if self.eve { Some(Eve) } + else if self.ferdie { Some(Ferdie) } + else if self.one { Some(One) } + else if self.two { Some(Two) } + else { None } + } + + /// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo` + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + + let password = if self.password_interactive { + #[cfg(not(target_os = "unknown"))] + { + Some(input_keystore_password()?.into()) + } + #[cfg(target_os = "unknown")] + None + } else if let Some(ref file) = self.password_filename { + Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) + } else if let Some(ref password) = self.password { + Some(password.clone().into()) + } else { + None + }; + + let path = self.keystore_path.clone().or( + config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) + ); + + config.keystore = KeystoreConfig::Path { + path: path.ok_or_else(|| "No `base_path` provided to create keystore path!".to_string())?, + password, + }; + + let keyring = self.get_keyring(); + let is_dev = self.shared_params.dev; + let is_light = self.light; + let is_authority = (self.validator || self.sentry || is_dev || keyring.is_some()) + && !is_light; + let role = + if is_light { + sc_service::Roles::LIGHT + } else if is_authority { + sc_service::Roles::AUTHORITY + } else { + sc_service::Roles::FULL + }; + + self.import_params.update_config(&mut config, role, is_dev)?; + + config.name = match (self.name.as_ref(), keyring) { + (Some(name), _) => name.to_string(), + (_, Some(keyring)) => keyring.to_string(), + (None, None) => generate_node_name(), + }; + if let Err(msg) = is_node_name_valid(&config.name) { + return Err(error::Error::Input( + format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", + config.name, + msg, + ) + )); + } + + // set sentry mode (i.e. act as an authority but **never** actively participate) + config.sentry_mode = self.sentry; + + config.offchain_worker = match (&self.offchain_worker, role) { + (OffchainWorkerEnabled::WhenValidating, sc_service::Roles::AUTHORITY) => true, + (OffchainWorkerEnabled::Always, _) => true, + (OffchainWorkerEnabled::Never, _) => false, + (OffchainWorkerEnabled::WhenValidating, _) => false, + }; + + config.roles = role; + config.disable_grandpa = self.no_grandpa; + + let client_id = config.client_id(); + let network_path = config + .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) + .expect("We provided a basepath"); + self.network_config.update_config( + &mut config, + network_path, + client_id, + is_dev, + )?; + + self.pool_config.update_config(&mut config)?; + + config.dev_key_seed = keyring + .map(|a| format!("//{}", a)).or_else(|| { + if is_dev && !is_light { + Some("//Alice".into()) + } else { + None + } + }); + + if config.rpc_http.is_none() || self.rpc_port.is_some() { + let rpc_interface: &str = interface_str(self.rpc_external, self.unsafe_rpc_external, self.validator)?; + config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), self.rpc_port)?); + } + if config.rpc_ws.is_none() || self.ws_port.is_some() { + let ws_interface: &str = interface_str(self.ws_external, self.unsafe_ws_external, self.validator)?; + config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), self.ws_port)?); + } + + config.rpc_ws_max_connections = self.ws_max_connections; + config.rpc_cors = self.rpc_cors.clone().unwrap_or_else(|| if is_dev { + log::warn!("Running in --dev mode, RPC CORS has been disabled."); + Cors::All + } else { + Cors::List(vec![ + "http://localhost:*".into(), + "http://127.0.0.1:*".into(), + "https://localhost:*".into(), + "https://127.0.0.1:*".into(), + "https://polkadot.js.org".into(), + "https://substrate-ui.parity.io".into(), + ]) + }).into(); + + // Override telemetry + if self.no_telemetry { + config.telemetry_endpoints = None; + } else if !self.telemetry_endpoints.is_empty() { + config.telemetry_endpoints = Some( + TelemetryEndpoints::new(self.telemetry_endpoints.clone()) + ); + } + + // Override prometheus + if self.no_prometheus { + config.prometheus_port = None; + } else if config.prometheus_port.is_none() { + let prometheus_interface: &str = if self.prometheus_external { "0.0.0.0" } else { "127.0.0.1" }; + config.prometheus_port = Some( + parse_address(&format!("{}:{}", prometheus_interface, 9615), self.prometheus_port)?); + } + + config.tracing_targets = self.import_params.tracing_targets.clone().into(); + config.tracing_receiver = self.import_params.tracing_receiver.clone().into(); + + // Imply forced authoring on --dev + config.force_authoring = self.shared_params.dev || self.force_authoring; + + Ok(()) + } + + /// Run the command that runs the node + pub fn run( + self, + config: Configuration, + new_light: FNL, + new_full: FNF, + version: &VersionInfo, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + FNL: FnOnce(Configuration) -> Result, + FNF: FnOnce(Configuration) -> Result, + SL: AbstractService + Unpin, + SF: AbstractService + Unpin, + { + info!("{}", version.name); + info!(" version {}", config.full_version()); + info!(" by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); + info!("Chain specification: {}", config.expect_chain_spec().name()); + info!("Node name: {}", config.name); + info!("Roles: {}", config.display_role()); + + match config.roles { + Roles::LIGHT => run_service_until_exit( + config, + new_light, + ), + _ => run_service_until_exit( + config, + new_full, + ), + } + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + self.shared_params.init(version) + } +} + +/// Check whether a node name is considered as valid +pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { + let name = _name.to_string(); + if name.chars().count() >= NODE_NAME_MAX_LENGTH { + return Err("Node name too long"); + } + + let invalid_chars = r"[\\.@]"; + let re = Regex::new(invalid_chars).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain invalid chars such as '.' and '@'"); + } + + let invalid_patterns = r"(https?:\\/+)?(www)+"; + let re = Regex::new(invalid_patterns).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain urls"); + } + + Ok(()) +} + +#[cfg(not(target_os = "unknown"))] +fn input_keystore_password() -> Result { + rpassword::read_password_from_tty(Some("Keystore password: ")) + .map_err(|e| format!("{:?}", e)) +} + +fn generate_node_name() -> String { + let result = loop { + let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); + let count = node_name.chars().count(); + + if count < NODE_NAME_MAX_LENGTH { + break node_name + } + }; + + result +} + +fn parse_address( + address: &str, + port: Option, +) -> Result { + let mut address: SocketAddr = address.parse().map_err( + |_| format!("Invalid address: {}", address) + )?; + if let Some(port) = port { + address.set_port(port); + } + + Ok(address) +} + +fn interface_str( + is_external: bool, + is_unsafe_external: bool, + is_validator: bool, +) -> Result<&'static str, error::Error> { + if is_external && is_validator { + return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ + used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ + the risks. See the options description for more information.".to_owned())); + } + + if is_external || is_unsafe_external { + log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ + available set of RPC methods."); + + Ok("0.0.0.0") + } else { + Ok("127.0.0.1") + } +} + +/// Default to verbosity level 0, if none is provided. +fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { + let pos = s.find(' '); + match pos { + None => { + Ok((s.to_owned(), 0)) + }, + Some(pos_) => { + let verbosity = s[pos_ + 1..].parse()?; + let url = s[..pos_].parse()?; + Ok((url, verbosity)) + } + } +} + +/// CORS setting +/// +/// The type is introduced to overcome `Option>` +/// handling of `structopt`. +#[derive(Clone, Debug)] +pub enum Cors { + /// All hosts allowed + All, + /// Only hosts on the list are allowed. + List(Vec), +} + +impl From for Option> { + fn from(cors: Cors) -> Self { + match cors { + Cors::All => None, + Cors::List(list) => Some(list), + } + } +} + +/// Parse cors origins +fn parse_cors(s: &str) -> Result> { + let mut is_all = false; + let mut origins = Vec::new(); + for part in s.split(',') { + match part { + "all" | "*" => { + is_all = true; + break; + }, + other => origins.push(other.to_owned()), + } + } + + Ok(if is_all { Cors::All } else { Cors::List(origins) }) +} + +#[cfg(test)] +mod tests { + use super::*; + use sc_service::config::DatabaseConfig; + + const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { + name: "node-test", + version: "0.1.0", + commit: "some_commit", + executable_name: "node-test", + description: "description", + author: "author", + support_url: "http://example.org", + copyright_start_year: 2020, + }; + + #[test] + fn tests_node_name_good() { + assert!(is_node_name_valid("short name").is_ok()); + } + + #[test] + fn tests_node_name_bad() { + assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); + assert!(is_node_name_valid("Dots.not.Ok").is_err()); + assert!(is_node_name_valid("http://visit.me").is_err()); + assert!(is_node_name_valid("https://visit.me").is_err()); + assert!(is_node_name_valid("www.visit.me").is_err()); + assert!(is_node_name_valid("email@domain").is_err()); + } + + #[test] + fn keystore_path_is_generated_correctly() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + Vec::new(), + None, + None, + None, + None::<()>, + ); + + for keystore_path in vec![None, Some("/keystore/path")] { + let args: Vec<&str> = vec![]; + let mut cli = RunCmd::from_iter(args); + cli.keystore_path = keystore_path.clone().map(PathBuf::from); + + let mut config = Configuration::default(); + config.config_dir = Some(PathBuf::from("/test/path")); + config.chain_spec = Some(chain_spec.clone()); + let chain_spec = chain_spec.clone(); + cli.update_config(&mut config, move |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + let expected_path = match keystore_path { + Some(path) => PathBuf::from(path), + None => PathBuf::from("/test/path/chains/test-id/keystore"), + }; + + assert_eq!(expected_path, config.keystore.path().unwrap().to_owned()); + } + } + + #[test] + fn ensure_load_spec_provide_defaults() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + vec!["boo".to_string()], + Some(TelemetryEndpoints::new(vec![("foo".to_string(), 42)])), + None, + None, + None::<()>, + ); + + let args: Vec<&str> = vec![]; + let cli = RunCmd::from_iter(args); + + let mut config = Configuration::from_version(TEST_VERSION_INFO); + cli.update_config(&mut config, |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + assert!(config.chain_spec.is_some()); + assert!(!config.network.boot_nodes.is_empty()); + assert!(config.telemetry_endpoints.is_some()); + } + + #[test] + fn ensure_update_config_for_running_node_provides_defaults() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + vec![], + None, + None, + None, + None::<()>, + ); + + let args: Vec<&str> = vec![]; + let cli = RunCmd::from_iter(args); + + let mut config = Configuration::from_version(TEST_VERSION_INFO); + cli.init(&TEST_VERSION_INFO).unwrap(); + cli.update_config(&mut config, |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + assert!(config.config_dir.is_some()); + assert!(config.database.is_some()); + if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { + assert!(cache_size.is_some()); + } else { + panic!("invalid config.database variant"); + } + assert!(!config.name.is_empty()); + assert!(config.network.config_path.is_some()); + assert!(!config.network.listen_addresses.is_empty()); + } +} diff --git a/client/cli/src/error.rs b/client/cli/src/error.rs index 074cb353c3a0a6d2a8cbd7653a156e1a169de04c..edc1adecc762c5f70c11958d1b6f9b0b93863904 100644 --- a/client/cli/src/error.rs +++ b/client/cli/src/error.rs @@ -49,6 +49,12 @@ impl std::convert::From for Error { } } +impl std::convert::From<&str> for Error { + fn from(s: &str) -> Error { + Error::Input(s.to_string()) + } +} + impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index 54a98a6ad3951ca9513b53ef52d4a1a0e950035c..e28edebd60def53ba34497d416a9bea7919ef983 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -19,127 +19,33 @@ #![warn(missing_docs)] #![warn(unused_extern_crates)] -#[macro_use] -mod traits; mod params; -mod execution_strategy; -pub mod error; -pub mod informant; +mod arg_enums; +mod error; mod runtime; -mod node_key; - -use sc_client_api::execution_extensions::ExecutionStrategies; -use sc_service::{ - config::{Configuration, DatabaseConfig, KeystoreConfig}, - ServiceBuilderCommand, - RuntimeGenesis, ChainSpecExtension, PruningMode, ChainSpec, - AbstractService, Roles as ServiceRoles, -}; +mod commands; + pub use sc_service::config::VersionInfo; -use sc_network::{ - self, - multiaddr::Protocol, - config::{ - NetworkConfiguration, TransportConfig, NonReservedPeerMode, - }, -}; - -use std::{ - io::Write, iter, fmt::Debug, fs, - net::{Ipv4Addr, SocketAddr}, path::PathBuf, -}; + +use std::io::Write; use regex::Regex; -use structopt::{StructOpt, clap}; +use structopt::{StructOpt, clap::{self, AppSettings}}; pub use structopt; -use params::{ - NetworkConfigurationParams, TransactionPoolParams, Cors, -}; -pub use params::{ - SharedParams, ImportParams, ExecutionStrategy, Subcommand, RunCmd, BuildSpecCmd, - ExportBlocksCmd, ImportBlocksCmd, CheckBlockCmd, PurgeChainCmd, RevertCmd, -}; -pub use traits::GetSharedParams; -use app_dirs::{AppInfo, AppDataType}; +pub use params::*; +pub use commands::*; +pub use arg_enums::*; +pub use error::*; use log::info; use lazy_static::lazy_static; -use sc_telemetry::TelemetryEndpoints; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; pub use crate::runtime::{run_until_exit, run_service_until_exit}; -use execution_strategy::*; -use names::{Generator, Name}; -use chrono::prelude::*; - -/// default sub directory to store network config -const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; -/// default sub directory to store database -const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; -/// default sub directory for the key store -const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; - -/// The maximum number of characters for a node name. -const NODE_NAME_MAX_LENGTH: usize = 32; - -fn get_chain_key(cli: &SharedParams) -> String { - match cli.chain { - Some(ref chain) => chain.clone(), - None => if cli.dev { "dev".into() } else { "".into() } - } -} - -fn generate_node_name() -> String { - let result = loop { - let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); - let count = node_name.chars().count(); - - if count < NODE_NAME_MAX_LENGTH { - break node_name - } - }; - - result -} - -/// Load spec to `Configuration` from shared params and spec factory. -pub fn load_spec<'a, G, E, F>( - mut config: &'a mut Configuration, - cli: &SharedParams, - factory: F, -) -> error::Result<&'a ChainSpec> where - G: RuntimeGenesis, - E: ChainSpecExtension, - F: FnOnce(&str) -> Result>, String>, -{ - let chain_key = get_chain_key(cli); - let spec = match factory(&chain_key)? { - Some(spec) => spec, - None => ChainSpec::from_json_file(PathBuf::from(chain_key))? - }; - - config.network.boot_nodes = spec.boot_nodes().to_vec(); - config.telemetry_endpoints = spec.telemetry_endpoints().clone(); - - config.chain_spec = Some(spec); - - Ok(config.chain_spec.as_ref().unwrap()) -} - -fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { - cli.base_path.clone() - .unwrap_or_else(|| - app_dirs::get_app_root( - AppDataType::UserData, - &AppInfo { - name: version.executable_name, - author: version.author - } - ).expect("app directories exist on all supported platforms; qed") - ) -} /// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_args()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. +/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from the command line arguments. Print the /// error message and quit the program in case of failure. @@ -152,7 +58,10 @@ where /// Helper function used to parse the command line arguments. This is the equivalent of /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from any iterator such as a `Vec` of your making. /// Print the error message and quit the program in case of failure. @@ -174,14 +83,22 @@ where .name(version.executable_name) .author(version.author) .about(version.description) - .version(full_version.as_str()); + .version(full_version.as_str()) + .settings(&[ + AppSettings::GlobalVersion, + AppSettings::ArgsNegateSubcommands, + AppSettings::SubcommandsNegateReqs, + ]); T::from_clap(&app.get_matches_from(iter)) } /// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `try_from_iter()` except that it takes a `VersionInfo` argument to provide the -/// name of the application, author, "about" and version. +/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from any iterator such as a `Vec` of your making. /// Print the error message and quit the program in case of failure. @@ -215,53 +132,6 @@ where Ok(T::from_clap(&matches)) } -/// A helper function that initializes and runs the node -pub fn run( - mut config: Configuration, - run_cmd: RunCmd, - new_light: FNL, - new_full: FNF, - spec_factory: F, - version: &VersionInfo, -) -> error::Result<()> -where - F: FnOnce(&str) -> Result>, String>, - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, -{ - init(&run_cmd.shared_params, version)?; - init_config(&mut config, &run_cmd.shared_params, version, spec_factory)?; - run_cmd.run(config, new_light, new_full, version) -} - -/// A helper function that initializes and runs any of the subcommand variants of `CoreParams`. -pub fn run_subcommand( - mut config: Configuration, - subcommand: Subcommand, - spec_factory: F, - builder: B, - version: &VersionInfo, -) -> error::Result<()> -where - F: FnOnce(&str) -> Result>, String>, - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, -{ - let shared_params = subcommand.get_shared_params(); - init(shared_params, version)?; - init_config(&mut config, shared_params, version, spec_factory)?; - subcommand.run(config, builder) -} - /// Initialize substrate. This must be done only once. /// /// This method: @@ -269,7 +139,7 @@ where /// 1. Set the panic handler /// 2. Raise the FD limit /// 3. Initialize the logger -pub fn init(shared_params: &SharedParams, version: &VersionInfo) -> error::Result<()> { +pub fn init(logger_pattern: &str, version: &VersionInfo) -> error::Result<()> { let full_version = sc_service::config::full_version_from_strs( version.version, version.commit @@ -277,424 +147,11 @@ pub fn init(shared_params: &SharedParams, version: &VersionInfo) -> error::Resul sp_panic_handler::set(version.support_url, &full_version); fdlimit::raise_fd_limit(); - init_logger(shared_params.log.as_ref().map(|v| v.as_ref()).unwrap_or("")); - - Ok(()) -} - -/// Initialize the given `config`. -/// -/// This will load the chain spec, set the `config_dir` and the `database_dir`. -pub fn init_config( - config: &mut Configuration, - shared_params: &SharedParams, - version: &VersionInfo, - spec_factory: F, -) -> error::Result<()> where - F: FnOnce(&str) -> Result>, String>, - G: RuntimeGenesis, - E: ChainSpecExtension, -{ - load_spec(config, shared_params, spec_factory)?; - - if config.config_dir.is_none() { - config.config_dir = Some(base_path(&shared_params, version)); - } - - if config.database.is_none() { - config.database = Some(DatabaseConfig::Path { - path: config - .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) - .expect("We provided a base_path/config_dir."), - cache_size: None, - }); - } - - Ok(()) -} - -/// Run the node -/// -/// Builds and runs either a full or a light node, depending on the `role` within the `Configuration`. -pub fn run_node( - config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, -) -> error::Result<()> -where - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, -{ - info!("{}", version.name); - info!(" version {}", config.full_version()); - info!(" by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); - info!("Chain specification: {}", config.expect_chain_spec().name()); - info!("Node name: {}", config.name); - info!("Roles: {}", display_role(&config)); - - match config.roles { - ServiceRoles::LIGHT => run_service_until_exit( - config, - new_light, - ), - _ => run_service_until_exit( - config, - new_full, - ), - } -} - -/// Returns a string displaying the node role, special casing the sentry mode -/// (returning `SENTRY`), since the node technically has an `AUTHORITY` role but -/// doesn't participate. -pub fn display_role(config: &Configuration) -> String { - if config.sentry_mode { - "SENTRY".to_string() - } else { - format!("{:?}", config.roles) - } -} - -/// Fill the given `PoolConfiguration` by looking at the cli parameters. -fn fill_transaction_pool_configuration( - options: &mut Configuration, - params: TransactionPoolParams, -) -> error::Result<()> { - // ready queue - options.transaction_pool.ready.count = params.pool_limit; - options.transaction_pool.ready.total_bytes = params.pool_kbytes * 1024; - - // future queue - let factor = 10; - options.transaction_pool.future.count = params.pool_limit / factor; - options.transaction_pool.future.total_bytes = params.pool_kbytes * 1024 / factor; + init_logger(logger_pattern); Ok(()) } -/// Fill the given `NetworkConfiguration` by looking at the cli parameters. -fn fill_network_configuration( - cli: NetworkConfigurationParams, - config_path: PathBuf, - config: &mut NetworkConfiguration, - client_id: String, - is_dev: bool, -) -> error::Result<()> { - config.boot_nodes.extend(cli.bootnodes.into_iter()); - config.config_path = Some(config_path.to_string_lossy().into()); - config.net_config_path = config.config_path.clone(); - - config.reserved_nodes.extend(cli.reserved_nodes.into_iter()); - if cli.reserved_only { - config.non_reserved_mode = NonReservedPeerMode::Deny; - } - - config.sentry_nodes.extend(cli.sentry_nodes.into_iter()); - - for addr in cli.listen_addr.iter() { - let addr = addr.parse().ok().ok_or(error::Error::InvalidListenMultiaddress)?; - config.listen_addresses.push(addr); - } - - if config.listen_addresses.is_empty() { - let port = match cli.port { - Some(port) => port, - None => 30333, - }; - - config.listen_addresses = vec![ - iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) - .chain(iter::once(Protocol::Tcp(port))) - .collect() - ]; - } - - config.client_version = client_id; - config.node_key = node_key::node_key_config(cli.node_key_params, &config.net_config_path)?; - - config.in_peers = cli.in_peers; - config.out_peers = cli.out_peers; - - config.transport = TransportConfig::Normal { - enable_mdns: !is_dev && !cli.no_mdns, - allow_private_ipv4: !cli.no_private_ipv4, - wasm_external_transport: None, - }; - - config.max_parallel_downloads = cli.max_parallel_downloads; - - Ok(()) -} - -#[cfg(not(target_os = "unknown"))] -fn input_keystore_password() -> Result { - rpassword::read_password_from_tty(Some("Keystore password: ")) - .map_err(|e| format!("{:?}", e)) -} - -/// Use in memory keystore config when it is not required at all. -pub fn fill_config_keystore_in_memory(config: &mut sc_service::Configuration) - -> Result<(), String> -{ - match &mut config.keystore { - cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, - _ => Err("Keystore config specified when it should not be!".into()), - } -} - -/// Fill the password field of the given config instance. -fn fill_config_keystore_password_and_path( - config: &mut sc_service::Configuration, - cli: &RunCmd, -) -> Result<(), String> { - let password = if cli.password_interactive { - #[cfg(not(target_os = "unknown"))] - { - Some(input_keystore_password()?.into()) - } - #[cfg(target_os = "unknown")] - None - } else if let Some(ref file) = cli.password_filename { - Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) - } else if let Some(ref password) = cli.password { - Some(password.clone().into()) - } else { - None - }; - - let path = cli.keystore_path.clone().or( - config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) - ); - - config.keystore = KeystoreConfig::Path { - path: path.ok_or_else(|| "No `base_path` provided to create keystore path!")?, - password, - }; - - Ok(()) -} - -/// Put block import CLI params into `config` object. -pub fn fill_import_params( - config: &mut Configuration, - cli: &ImportParams, - role: sc_service::Roles, - is_dev: bool, -) -> error::Result<()> -where - G: RuntimeGenesis, -{ - if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { - *cache_size = Some(cli.database_cache_size); - } - - config.state_cache_size = cli.state_cache_size; - - // by default we disable pruning if the node is an authority (i.e. - // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the - // node is an authority and pruning is enabled explicitly, then we error - // unless `unsafe_pruning` is set. - config.pruning = match &cli.pruning { - Some(ref s) if s == "archive" => PruningMode::ArchiveAll, - None if role == sc_service::Roles::AUTHORITY => PruningMode::ArchiveAll, - None => PruningMode::default(), - Some(s) => { - if role == sc_service::Roles::AUTHORITY && !cli.unsafe_pruning { - return Err(error::Error::Input( - "Validators should run with state pruning disabled (i.e. archive). \ - You can ignore this check with `--unsafe-pruning`.".to_string() - )); - } - - PruningMode::keep_blocks(s.parse() - .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? - ) - }, - }; - - config.wasm_method = cli.wasm_method.into(); - - let exec = &cli.execution_strategies; - let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { - exec.execution.unwrap_or(if strat == default && is_dev { - ExecutionStrategy::Native - } else { - strat - }).into() - }; - - config.execution_strategies = ExecutionStrategies { - syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), - importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), - block_construction: - exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION), - offchain_worker: - exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), - other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), - }; - Ok(()) -} - -/// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo` -pub fn update_config_for_running_node( - mut config: &mut Configuration, - cli: RunCmd, -) -> error::Result<()> -where - G: RuntimeGenesis, -{ - fill_config_keystore_password_and_path(&mut config, &cli)?; - - let keyring = cli.get_keyring(); - let is_dev = cli.shared_params.dev; - let is_light = cli.light; - let is_authority = (cli.validator || cli.sentry || is_dev || keyring.is_some()) - && !is_light; - let role = - if is_light { - sc_service::Roles::LIGHT - } else if is_authority { - sc_service::Roles::AUTHORITY - } else { - sc_service::Roles::FULL - }; - - fill_import_params(&mut config, &cli.import_params, role, is_dev)?; - - config.name = match (cli.name.as_ref(), keyring) { - (Some(name), _) => name.to_string(), - (_, Some(keyring)) => keyring.to_string(), - (None, None) => generate_node_name(), - }; - if let Err(msg) = node_key::is_node_name_valid(&config.name) { - return Err(error::Error::Input( - format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", - config.name, - msg, - ) - )); - } - - // set sentry mode (i.e. act as an authority but **never** actively participate) - config.sentry_mode = cli.sentry; - - config.offchain_worker = match (cli.offchain_worker, role) { - (params::OffchainWorkerEnabled::WhenValidating, sc_service::Roles::AUTHORITY) => true, - (params::OffchainWorkerEnabled::Always, _) => true, - (params::OffchainWorkerEnabled::Never, _) => false, - (params::OffchainWorkerEnabled::WhenValidating, _) => false, - }; - - config.roles = role; - config.disable_grandpa = cli.no_grandpa; - - let client_id = config.client_id(); - fill_network_configuration( - cli.network_config, - config.in_chain_config_dir(DEFAULT_NETWORK_CONFIG_PATH).expect("We provided a basepath"), - &mut config.network, - client_id, - is_dev, - )?; - - fill_transaction_pool_configuration(&mut config, cli.pool_config)?; - - config.dev_key_seed = keyring - .map(|a| format!("//{}", a)).or_else(|| { - if is_dev && !is_light { - Some("//Alice".into()) - } else { - None - } - }); - - if config.rpc_http.is_none() || cli.rpc_port.is_some() { - let rpc_interface: &str = interface_str(cli.rpc_external, cli.unsafe_rpc_external, cli.validator)?; - config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), cli.rpc_port)?); - } - if config.rpc_ws.is_none() || cli.ws_port.is_some() { - let ws_interface: &str = interface_str(cli.ws_external, cli.unsafe_ws_external, cli.validator)?; - config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), cli.ws_port)?); - } - - if config.grafana_port.is_none() || cli.grafana_port.is_some() { - let grafana_interface: &str = if cli.grafana_external { "0.0.0.0" } else { "127.0.0.1" }; - config.grafana_port = Some( - parse_address(&format!("{}:{}", grafana_interface, 9955), cli.grafana_port)? - ); - } - - config.rpc_ws_max_connections = cli.ws_max_connections; - config.rpc_cors = cli.rpc_cors.unwrap_or_else(|| if is_dev { - log::warn!("Running in --dev mode, RPC CORS has been disabled."); - Cors::All - } else { - Cors::List(vec![ - "http://localhost:*".into(), - "http://127.0.0.1:*".into(), - "https://localhost:*".into(), - "https://127.0.0.1:*".into(), - "https://polkadot.js.org".into(), - "https://substrate-ui.parity.io".into(), - ]) - }).into(); - - // Override telemetry - if cli.no_telemetry { - config.telemetry_endpoints = None; - } else if !cli.telemetry_endpoints.is_empty() { - config.telemetry_endpoints = Some(TelemetryEndpoints::new(cli.telemetry_endpoints)); - } - - config.tracing_targets = cli.import_params.tracing_targets.into(); - config.tracing_receiver = cli.import_params.tracing_receiver.into(); - - // Imply forced authoring on --dev - config.force_authoring = cli.shared_params.dev || cli.force_authoring; - - Ok(()) -} - -fn interface_str( - is_external: bool, - is_unsafe_external: bool, - is_validator: bool, -) -> Result<&'static str, error::Error> { - if is_external && is_validator { - return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ - used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ - the risks. See the options description for more information.".to_owned())); - } - - if is_external || is_unsafe_external { - log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ - available set of RPC methods."); - - Ok("0.0.0.0") - } else { - Ok("127.0.0.1") - } -} - -fn parse_address( - address: &str, - port: Option, -) -> Result { - let mut address: SocketAddr = address.parse().map_err( - |_| format!("Invalid address: {}", address) - )?; - if let Some(port) = port { - address.set_port(port); - } - - Ok(address) -} - /// Initialize the logger pub fn init_logger(pattern: &str) { use ansi_term::Colour; @@ -764,116 +221,3 @@ fn kill_color(s: &str) -> String { } RE.replace_all(s, "").to_string() } - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { - name: "node-test", - version: "0.1.0", - commit: "some_commit", - executable_name: "node-test", - description: "description", - author: "author", - support_url: "http://example.org", - copyright_start_year: 2020, - }; - - #[test] - fn keystore_path_is_generated_correctly() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - Vec::new(), - None, - None, - None, - None::<()>, - ); - - for keystore_path in vec![None, Some("/keystore/path")] { - let args: Vec<&str> = vec![]; - let mut run_cmds = RunCmd::from_iter(args); - run_cmds.keystore_path = keystore_path.clone().map(PathBuf::from); - - let mut node_config = Configuration::default(); - node_config.config_dir = Some(PathBuf::from("/test/path")); - node_config.chain_spec = Some(chain_spec.clone()); - update_config_for_running_node( - &mut node_config, - run_cmds.clone(), - ).unwrap(); - - let expected_path = match keystore_path { - Some(path) => PathBuf::from(path), - None => PathBuf::from("/test/path/chains/test-id/keystore"), - }; - - assert_eq!(expected_path, node_config.keystore.path().unwrap().to_owned()); - } - } - - #[test] - fn ensure_load_spec_provide_defaults() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - vec!["boo".to_string()], - Some(TelemetryEndpoints::new(vec![("foo".to_string(), 42)])), - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::new(TEST_VERSION_INFO); - load_spec(&mut config, &cli.shared_params, |_| Ok(Some(chain_spec))).unwrap(); - - assert!(config.chain_spec.is_some()); - assert!(!config.network.boot_nodes.is_empty()); - assert!(config.telemetry_endpoints.is_some()); - } - - #[test] - fn ensure_update_config_for_running_node_provides_defaults() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - vec![], - None, - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::new(TEST_VERSION_INFO); - init(&cli.shared_params, &TEST_VERSION_INFO).unwrap(); - init_config( - &mut config, - &cli.shared_params, - &TEST_VERSION_INFO, - |_| Ok(Some(chain_spec)), - ).unwrap(); - update_config_for_running_node(&mut config, cli).unwrap(); - - assert!(config.config_dir.is_some()); - assert!(config.database.is_some()); - if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { - assert!(cache_size.is_some()); - } else { - panic!("invalid config.database variant"); - } - assert!(!config.name.is_empty()); - assert!(config.network.config_path.is_some()); - assert!(!config.network.listen_addresses.is_empty()); - } -} diff --git a/client/cli/src/node_key.rs b/client/cli/src/node_key.rs deleted file mode 100644 index 4401481ca56ce1ca171cc8ab9c9ade5ee5874ee7..0000000000000000000000000000000000000000 --- a/client/cli/src/node_key.rs +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use sc_network::{ - self, - config::{ - NodeKeyConfig, - }, -}; -use sp_core::H256; -use regex::Regex; -use std::{path::{Path, PathBuf}, str::FromStr}; -use crate::error; -use crate::params::{NodeKeyParams, NodeKeyType}; - -/// The file name of the node's Ed25519 secret key inside the chain-specific -/// network config directory, if neither `--node-key` nor `--node-key-file` -/// is specified in combination with `--node-key-type=ed25519`. -const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; - -/// Check whether a node name is considered as valid -pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { - let name = _name.to_string(); - if name.chars().count() >= crate::NODE_NAME_MAX_LENGTH { - return Err("Node name too long"); - } - - let invalid_chars = r"[\\.@]"; - let re = Regex::new(invalid_chars).unwrap(); - if re.is_match(&name) { - return Err("Node name should not contain invalid chars such as '.' and '@'"); - } - - let invalid_patterns = r"(https?:\\/+)?(www)+"; - let re = Regex::new(invalid_patterns).unwrap(); - if re.is_match(&name) { - return Err("Node name should not contain urls"); - } - - Ok(()) -} - -/// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context -/// of an optional network config storage directory. -pub fn node_key_config

(params: NodeKeyParams, net_config_dir: &Option

) - -> error::Result -where - P: AsRef -{ - match params.node_key_type { - NodeKeyType::Ed25519 => - params.node_key.as_ref().map(parse_ed25519_secret).unwrap_or_else(|| - Ok(params.node_key_file - .or_else(|| net_config_file(net_config_dir, NODE_KEY_ED25519_FILE)) - .map(sc_network::config::Secret::File) - .unwrap_or(sc_network::config::Secret::New))) - .map(NodeKeyConfig::Ed25519) - } -} - -/// Create an error caused by an invalid node key argument. -fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { - error::Error::Input(format!("Invalid node key: {}", e)) -} - -/// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. -fn parse_ed25519_secret(hex: &String) -> error::Result { - H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| - sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) - .map(sc_network::config::Secret::Input) - .map_err(invalid_node_key)) -} - -fn net_config_file

(net_config_dir: &Option

, name: &str) -> Option -where - P: AsRef -{ - net_config_dir.as_ref().map(|d| d.as_ref().join(name)) -} - -#[cfg(test)] -mod tests { - use sc_network::config::identity::ed25519; - use super::*; - - #[test] - fn tests_node_name_good() { - assert!(is_node_name_valid("short name").is_ok()); - } - - #[test] - fn tests_node_name_bad() { - assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); - assert!(is_node_name_valid("Dots.not.Ok").is_err()); - assert!(is_node_name_valid("http://visit.me").is_err()); - assert!(is_node_name_valid("https://visit.me").is_err()); - assert!(is_node_name_valid("www.visit.me").is_err()); - assert!(is_node_name_valid("email@domain").is_err()); - } - - #[test] - fn test_node_key_config_input() { - fn secret_input(net_config_dir: Option) -> error::Result<()> { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - let sk = match node_key_type { - NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() - }; - let params = NodeKeyParams { - node_key_type, - node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), - node_key_file: None - }; - node_key_config(params, &net_config_dir).and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) - if node_key_type == NodeKeyType::Ed25519 && - &sk[..] == ski.as_ref() => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(secret_input(None).is_ok()); - assert!(secret_input(Some("x".to_string())).is_ok()); - } - - #[test] - fn test_node_key_config_file() { - fn secret_file(net_config_dir: Option) -> error::Result<()> { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; - let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); - let params = NodeKeyParams { - node_key_type, - node_key: None, - node_key_file: Some(file.clone()) - }; - node_key_config(params, &net_config_dir).and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(secret_file(None).is_ok()); - assert!(secret_file(Some("x".to_string())).is_ok()); - } - - #[test] - fn test_node_key_config_default() { - fn with_def_params(f: F) -> error::Result<()> - where - F: Fn(NodeKeyParams) -> error::Result<()> - { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - f(NodeKeyParams { - node_key_type, - node_key: None, - node_key_file: None - }) - }) - } - - fn no_config_dir() -> error::Result<()> { - with_def_params(|params| { - let typ = params.node_key_type; - node_key_config::(params, &None) - .and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::New) - if typ == NodeKeyType::Ed25519 => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - fn some_config_dir(net_config_dir: String) -> error::Result<()> { - with_def_params(|params| { - let dir = PathBuf::from(net_config_dir.clone()); - let typ = params.node_key_type; - node_key_config(params, &Some(net_config_dir.clone())) - .and_then(move |c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if typ == NodeKeyType::Ed25519 && - f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(no_config_dir().is_ok()); - assert!(some_config_dir("x".to_string()).is_ok()); - } -} diff --git a/client/cli/src/params.rs b/client/cli/src/params.rs deleted file mode 100644 index a99d887d25fa5eaf9e5db19b8acc3965f5d53a29..0000000000000000000000000000000000000000 --- a/client/cli/src/params.rs +++ /dev/null @@ -1,1188 +0,0 @@ -// Copyright 2018-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use std::{str::FromStr, path::PathBuf}; -use structopt::{StructOpt, clap::arg_enum}; -use sc_service::{ - AbstractService, Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, - config::DatabaseConfig, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; -use crate::VersionInfo; -use crate::error; -use std::fmt::Debug; -use log::info; -use sc_network::config::build_multiaddr; -use std::io; -use std::fs; -use std::io::{Read, Write, Seek}; -use sp_runtime::generic::BlockId; -use crate::runtime::run_until_exit; -use crate::node_key::node_key_config; -use crate::execution_strategy::*; - -pub use crate::execution_strategy::ExecutionStrategy; - -impl Into for ExecutionStrategy { - fn into(self) -> sc_client_api::ExecutionStrategy { - match self { - ExecutionStrategy::Native => sc_client_api::ExecutionStrategy::NativeWhenPossible, - ExecutionStrategy::Wasm => sc_client_api::ExecutionStrategy::AlwaysWasm, - ExecutionStrategy::Both => sc_client_api::ExecutionStrategy::Both, - ExecutionStrategy::NativeElseWasm => sc_client_api::ExecutionStrategy::NativeElseWasm, - } - } -} - -arg_enum! { - /// How to execute Wasm runtime code - #[allow(missing_docs)] - #[derive(Debug, Clone, Copy)] - pub enum WasmExecutionMethod { - // Uses an interpreter. - Interpreted, - // Uses a compiled runtime. - Compiled, - } -} - -impl WasmExecutionMethod { - /// Returns list of variants that are not disabled by feature flags. - fn enabled_variants() -> Vec<&'static str> { - Self::variants() - .iter() - .cloned() - .filter(|&name| cfg!(feature = "wasmtime") || name != "Compiled") - .collect() - } -} - -impl Into for WasmExecutionMethod { - fn into(self) -> sc_service::config::WasmExecutionMethod { - match self { - WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, - #[cfg(feature = "wasmtime")] - WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, - #[cfg(not(feature = "wasmtime"))] - WasmExecutionMethod::Compiled => panic!( - "Substrate must be compiled with \"wasmtime\" feature for compiled Wasm execution" - ), - } - } -} - -arg_enum! { - /// Whether off-chain workers are enabled. - #[allow(missing_docs)] - #[derive(Debug, Clone)] - pub enum OffchainWorkerEnabled { - Always, - Never, - WhenValidating, - } -} - -/// Shared parameters used by all `CoreParams`. -#[derive(Debug, StructOpt, Clone)] -pub struct SharedParams { - /// Specify the chain specification (one of dev, local or staging). - #[structopt(long = "chain", value_name = "CHAIN_SPEC")] - pub chain: Option, - - /// Specify the development chain. - #[structopt(long = "dev")] - pub dev: bool, - - /// Specify custom base path. - #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] - pub base_path: Option, - - /// Sets a custom logging filter. - #[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")] - pub log: Option, -} - -/// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] -pub struct ImportParams { - /// Specify the state pruning mode, a number of blocks to keep or 'archive'. - /// - /// Default is to keep all block states if the node is running as a - /// validator (i.e. 'archive'), otherwise state is only kept for the last - /// 256 blocks. - #[structopt(long = "pruning", value_name = "PRUNING_MODE")] - pub pruning: Option, - - /// Force start with unsafe pruning settings. - /// - /// When running as a validator it is highly recommended to disable state - /// pruning (i.e. 'archive') which is the default. The node will refuse to - /// start as a validator if pruning is enabled unless this option is set. - #[structopt(long = "unsafe-pruning")] - pub unsafe_pruning: bool, - - /// Method for executing Wasm runtime code. - #[structopt( - long = "wasm-execution", - value_name = "METHOD", - possible_values = &WasmExecutionMethod::enabled_variants(), - case_insensitive = true, - default_value = "Interpreted" - )] - pub wasm_method: WasmExecutionMethod, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub execution_strategies: ExecutionStrategies, - - /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB", default_value = "1024")] - pub database_cache_size: u32, - - /// Specify the state cache size. - #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] - pub state_cache_size: usize, - - /// Comma separated list of targets for tracing - #[structopt(long = "tracing-targets", value_name = "TARGETS")] - pub tracing_targets: Option, - - /// Receiver to process tracing messages - #[structopt( - long = "tracing-receiver", - value_name = "RECEIVER", - possible_values = &TracingReceiver::variants(), - case_insensitive = true, - default_value = "Log" - )] - pub tracing_receiver: TracingReceiver, -} - -/// Parameters used to create the network configuration. -#[derive(Debug, StructOpt, Clone)] -pub struct NetworkConfigurationParams { - /// Specify a list of bootnodes. - #[structopt(long = "bootnodes", value_name = "URL")] - pub bootnodes: Vec, - - /// Specify a list of reserved node addresses. - #[structopt(long = "reserved-nodes", value_name = "URL")] - pub reserved_nodes: Vec, - - /// Whether to only allow connections to/from reserved nodes. - /// - /// If you are a validator your node might still connect to other validator - /// nodes regardless of whether they are defined as reserved nodes. - #[structopt(long = "reserved-only")] - pub reserved_only: bool, - - /// Specify a list of sentry node public addresses. - #[structopt( - long = "sentry-nodes", - value_name = "URL", - conflicts_with_all = &[ "sentry" ] - )] - pub sentry_nodes: Vec, - - /// Listen on this multiaddress. - #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] - pub listen_addr: Vec, - - /// Specify p2p protocol TCP port. - /// - /// Only used if --listen-addr is not specified. - #[structopt(long = "port", value_name = "PORT")] - pub port: Option, - - /// Allow connecting to private IPv4 addresses (as specified in - /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with - /// `--reserved-nodes` or `--bootnodes`. - #[structopt(long = "no-private-ipv4")] - pub no_private_ipv4: bool, - - /// Specify the number of outgoing connections we're trying to maintain. - #[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")] - pub out_peers: u32, - - /// Specify the maximum number of incoming connections we're accepting. - #[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")] - pub in_peers: u32, - - /// Disable mDNS discovery. - /// - /// By default, the network will use mDNS to discover other nodes on the - /// local network. This disables it. Automatically implied when using --dev. - #[structopt(long = "no-mdns")] - pub no_mdns: bool, - - /// Maximum number of peers to ask the same blocks in parallel. - /// - /// This allows downlading announced blocks from multiple peers. Decrease to save - /// traffic and risk increased latency. - #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] - pub max_parallel_downloads: u32, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub node_key_params: NodeKeyParams, -} - -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum NodeKeyType { - Ed25519 - } -} - -/// Parameters used to create the `NodeKeyConfig`, which determines the keypair -/// used for libp2p networking. -#[derive(Debug, StructOpt, Clone)] -pub struct NodeKeyParams { - /// The secret key to use for libp2p networking. - /// - /// The value is a string that is parsed according to the choice of - /// `--node-key-type` as follows: - /// - /// `ed25519`: - /// The value is parsed as a hex-encoded Ed25519 32 bytes secret key, - /// i.e. 64 hex characters. - /// - /// The value of this option takes precedence over `--node-key-file`. - /// - /// WARNING: Secrets provided as command-line arguments are easily exposed. - /// Use of this option should be limited to development and testing. To use - /// an externally managed secret key, use `--node-key-file` instead. - #[structopt(long = "node-key", value_name = "KEY")] - pub node_key: Option, - - /// The type of secret key to use for libp2p networking. - /// - /// The secret key of the node is obtained as follows: - /// - /// * If the `--node-key` option is given, the value is parsed as a secret key - /// according to the type. See the documentation for `--node-key`. - /// - /// * If the `--node-key-file` option is given, the secret key is read from the - /// specified file. See the documentation for `--node-key-file`. - /// - /// * Otherwise, the secret key is read from a file with a predetermined, - /// type-specific name from the chain-specific network config directory - /// inside the base directory specified by `--base-dir`. If this file does - /// not exist, it is created with a newly generated secret key of the - /// chosen type. - /// - /// The node's secret key determines the corresponding public key and hence the - /// node's peer ID in the context of libp2p. - #[structopt( - long = "node-key-type", - value_name = "TYPE", - possible_values = &NodeKeyType::variants(), - case_insensitive = true, - default_value = "Ed25519" - )] - pub node_key_type: NodeKeyType, - - /// The file from which to read the node's secret key to use for libp2p networking. - /// - /// The contents of the file are parsed according to the choice of `--node-key-type` - /// as follows: - /// - /// `ed25519`: - /// The file must contain an unencoded 32 bytes Ed25519 secret key. - /// - /// If the file does not exist, it is created with a newly generated secret key of - /// the chosen type. - #[structopt(long = "node-key-file", value_name = "FILE")] - pub node_key_file: Option, -} - -/// Parameters used to create the pool configuration. -#[derive(Debug, StructOpt, Clone)] -pub struct TransactionPoolParams { - /// Maximum number of transactions in the transaction pool. - #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] - pub pool_limit: usize, - /// Maximum number of kilobytes of all transactions stored in the pool. - #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] - pub pool_kbytes: usize, -} - -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum TracingReceiver { - Log, - Telemetry, - Grafana, - } -} - -impl Into for TracingReceiver { - fn into(self) -> sc_tracing::TracingReceiver { - match self { - TracingReceiver::Log => sc_tracing::TracingReceiver::Log, - TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry, - TracingReceiver::Grafana => sc_tracing::TracingReceiver::Grafana, - } - } -} - -/// Execution strategies parameters. -#[derive(Debug, StructOpt, Clone)] -pub struct ExecutionStrategies { - /// The means of execution used when calling into the runtime while syncing blocks. - #[structopt( - long = "execution-syncing", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_SYNCING.as_str(), - )] - pub execution_syncing: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while importing blocks. - #[structopt( - long = "execution-import-block", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_IMPORT_BLOCK.as_str(), - )] - pub execution_import_block: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while constructing blocks. - #[structopt( - long = "execution-block-construction", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_BLOCK_CONSTRUCTION.as_str(), - )] - pub execution_block_construction: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while using an off-chain worker. - #[structopt( - long = "execution-offchain-worker", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_OFFCHAIN_WORKER.as_str(), - )] - pub execution_offchain_worker: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks. - #[structopt( - long = "execution-other", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_OTHER.as_str(), - )] - pub execution_other: ExecutionStrategy, - - /// The execution strategy that should be used by all execution contexts. - #[structopt( - long = "execution", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - conflicts_with_all = &[ - "execution-other", - "execution-offchain-worker", - "execution-block-construction", - "execution-import-block", - "execution-syncing", - ] - )] - pub execution: Option, -} - -/// The `run` command used to run a node. -#[derive(Debug, StructOpt, Clone)] -pub struct RunCmd { - /// Enable validator mode. - /// - /// The node will be started with the authority role and actively - /// participate in any consensus task that it can (e.g. depending on - /// availability of local keys). - #[structopt( - long = "validator", - conflicts_with_all = &[ "sentry" ] - )] - pub validator: bool, - - /// Enable sentry mode. - /// - /// The node will be started with the authority role and participate in - /// consensus tasks as an "observer", it will never actively participate - /// regardless of whether it could (e.g. keys are available locally). This - /// mode is useful as a secure proxy for validators (which would run - /// detached from the network), since we want this node to participate in - /// the full consensus protocols in order to have all needed consensus data - /// available to relay to private nodes. - #[structopt( - long = "sentry", - conflicts_with_all = &[ "validator", "light" ] - )] - pub sentry: bool, - - /// Disable GRANDPA voter when running in validator mode, otherwise disables the GRANDPA observer. - #[structopt(long = "no-grandpa")] - pub no_grandpa: bool, - - /// Experimental: Run in light client mode. - #[structopt(long = "light", conflicts_with = "sentry")] - pub light: bool, - - /// Listen to all RPC interfaces. - /// - /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy - /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. - /// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks. - #[structopt(long = "rpc-external")] - pub rpc_external: bool, - - /// Listen to all RPC interfaces. - /// - /// Same as `--rpc-external`. - #[structopt(long = "unsafe-rpc-external")] - pub unsafe_rpc_external: bool, - - /// Listen to all Websocket interfaces. - /// - /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy - /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. - /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. - #[structopt(long = "ws-external")] - pub ws_external: bool, - - /// Listen to all Websocket interfaces. - /// - /// Same as `--ws-external`. - #[structopt(long = "unsafe-ws-external")] - pub unsafe_ws_external: bool, - - /// Listen to all Grafana data source interfaces. - /// - /// Default is local. - #[structopt(long = "grafana-external")] - pub grafana_external: bool, - - /// Specify HTTP RPC server TCP port. - #[structopt(long = "rpc-port", value_name = "PORT")] - pub rpc_port: Option, - - /// Specify WebSockets RPC server TCP port. - #[structopt(long = "ws-port", value_name = "PORT")] - pub ws_port: Option, - - /// Maximum number of WS RPC server connections. - #[structopt(long = "ws-max-connections", value_name = "COUNT")] - pub ws_max_connections: Option, - - /// Specify browser Origins allowed to access the HTTP & WS RPC servers. - /// - /// A comma-separated list of origins (protocol://domain or special `null` - /// value). Value of `all` will disable origin validation. Default is to - /// allow localhost, https://polkadot.js.org and - /// https://substrate-ui.parity.io origins. When running in --dev mode the - /// default is to allow all origins. - #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] - pub rpc_cors: Option, - - /// Specify Grafana data source server TCP Port. - #[structopt(long = "grafana-port", value_name = "PORT")] - pub grafana_port: Option, - - /// The human-readable name for this node. - /// - /// The node name will be reported to the telemetry server, if enabled. - #[structopt(long = "name", value_name = "NAME")] - pub name: Option, - - /// Disable connecting to the Substrate telemetry server. - /// - /// Telemetry is on by default on global chains. - #[structopt(long = "no-telemetry")] - pub no_telemetry: bool, - - /// The URL of the telemetry server to connect to. - /// - /// This flag can be passed multiple times as a mean to specify multiple - /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting - /// the least verbosity. If no verbosity level is specified the default is - /// 0. - #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] - pub telemetry_endpoints: Vec<(String, u8)>, - - /// Should execute offchain workers on every block. - /// - /// By default it's only enabled for nodes that are authoring new blocks. - #[structopt( - long = "offchain-worker", - value_name = "ENABLED", - possible_values = &OffchainWorkerEnabled::variants(), - case_insensitive = true, - default_value = "WhenValidating" - )] - pub offchain_worker: OffchainWorkerEnabled, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub network_config: NetworkConfigurationParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub pool_config: TransactionPoolParams, - - /// Shortcut for `--name Alice --validator` with session keys for `Alice` added to keystore. - #[structopt(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] - pub alice: bool, - - /// Shortcut for `--name Bob --validator` with session keys for `Bob` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] - pub bob: bool, - - /// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] - pub charlie: bool, - - /// Shortcut for `--name Dave --validator` with session keys for `Dave` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] - pub dave: bool, - - /// Shortcut for `--name Eve --validator` with session keys for `Eve` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] - pub eve: bool, - - /// Shortcut for `--name Ferdie --validator` with session keys for `Ferdie` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] - pub ferdie: bool, - - /// Shortcut for `--name One --validator` with session keys for `One` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] - pub one: bool, - - /// Shortcut for `--name Two --validator` with session keys for `Two` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] - pub two: bool, - - /// Enable authoring even when offline. - #[structopt(long = "force-authoring")] - pub force_authoring: bool, - - /// Specify custom keystore path. - #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] - pub keystore_path: Option, - - /// Use interactive shell for entering the password used by the keystore. - #[structopt( - long = "password-interactive", - conflicts_with_all = &[ "password", "password-filename" ] - )] - pub password_interactive: bool, - - /// Password used by the keystore. - #[structopt( - long = "password", - conflicts_with_all = &[ "password-interactive", "password-filename" ] - )] - pub password: Option, - - /// File that contains the password used by the keystore. - #[structopt( - long = "password-filename", - value_name = "PATH", - parse(from_os_str), - conflicts_with_all = &[ "password-interactive", "password" ] - )] - pub password_filename: Option -} - -impl RunCmd { - /// Get the `Sr25519Keyring` matching one of the flag - pub fn get_keyring(&self) -> Option { - use sp_keyring::Sr25519Keyring::*; - - if self.alice { Some(Alice) } - else if self.bob { Some(Bob) } - else if self.charlie { Some(Charlie) } - else if self.dave { Some(Dave) } - else if self.eve { Some(Eve) } - else if self.ferdie { Some(Ferdie) } - else if self.one { Some(One) } - else if self.two { Some(Two) } - else { None } - } -} - -/// Default to verbosity level 0, if none is provided. -fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { - let pos = s.find(' '); - match pos { - None => { - Ok((s.to_owned(), 0)) - }, - Some(pos_) => { - let verbosity = s[pos_ + 1..].parse()?; - let url = s[..pos_].parse()?; - Ok((url, verbosity)) - } - } -} - -/// CORS setting -/// -/// The type is introduced to overcome `Option>` -/// handling of `structopt`. -#[derive(Clone, Debug)] -pub enum Cors { - /// All hosts allowed - All, - /// Only hosts on the list are allowed. - List(Vec), -} - -impl From for Option> { - fn from(cors: Cors) -> Self { - match cors { - Cors::All => None, - Cors::List(list) => Some(list), - } - } -} - -/// Parse cors origins -fn parse_cors(s: &str) -> Result> { - let mut is_all = false; - let mut origins = Vec::new(); - for part in s.split(',') { - match part { - "all" | "*" => { - is_all = true; - break; - }, - other => origins.push(other.to_owned()), - } - } - - Ok(if is_all { Cors::All } else { Cors::List(origins) }) -} - -/// The `build-spec` command used to build a specification. -#[derive(Debug, StructOpt, Clone)] -pub struct BuildSpecCmd { - /// Force raw genesis storage output. - #[structopt(long = "raw")] - pub raw: bool, - - /// Disable adding the default bootnode to the specification. - /// - /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the - /// specification when no bootnode exists. - #[structopt(long = "disable-default-bootnode")] - pub disable_default_bootnode: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub node_key_params: NodeKeyParams, -} - -/// Wrapper type of `String` which holds an arbitary sized unsigned integer formatted as decimal. -#[derive(Debug, Clone)] -pub struct BlockNumber(String); - -impl FromStr for BlockNumber { - type Err = String; - - fn from_str(block_number: &str) -> Result { - if block_number.chars().any(|d| !d.is_digit(10)) { - Err(format!( - "Invalid block number: {}, expected decimal formatted unsigned integer", - block_number - )) - } else { - Ok(Self(block_number.to_owned())) - } - } -} - -impl BlockNumber { - /// Wrapper on top of `std::str::parse` but with `Error` as a `String` - /// - /// See `https://doc.rust-lang.org/std/primitive.str.html#method.parse` for more elaborate - /// documentation. - pub fn parse(&self) -> Result - where - N: FromStr, - N::Err: std::fmt::Debug, - { - self.0 - .parse() - .map_err(|e| format!("BlockNumber: {} parsing failed because of {:?}", self.0, e)) - } -} - -/// The `export-blocks` command used to export blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct ExportBlocksCmd { - /// Output file name or stdout if unspecified. - #[structopt(parse(from_os_str))] - pub output: Option, - - /// Specify starting block number. - /// - /// Default is 1. - #[structopt(long = "from", value_name = "BLOCK")] - pub from: Option, - - /// Specify last block number. - /// - /// Default is best block. - #[structopt(long = "to", value_name = "BLOCK")] - pub to: Option, - - /// Use JSON output rather than binary. - #[structopt(long = "json")] - pub json: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// The `import-blocks` command used to import blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct ImportBlocksCmd { - /// Input file or stdin if unspecified. - #[structopt(parse(from_os_str))] - pub input: Option, - - /// The default number of 64KB pages to ever allocate for Wasm execution. - /// - /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] - pub default_heap_pages: Option, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, -} - -/// The `check-block` command used to validate blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct CheckBlockCmd { - /// Block hash or number - #[structopt(value_name = "HASH or NUMBER")] - pub input: String, - - /// The default number of 64KB pages to ever allocate for Wasm execution. - /// - /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] - pub default_heap_pages: Option, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, -} - -/// The `revert` command used revert the chain to a previous state. -#[derive(Debug, StructOpt, Clone)] -pub struct RevertCmd { - /// Number of blocks to revert. - #[structopt(default_value = "256")] - pub num: BlockNumber, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// The `purge-chain` command used to remove the whole chain. -#[derive(Debug, StructOpt, Clone)] -pub struct PurgeChainCmd { - /// Skip interactive prompt by answering yes automatically. - #[structopt(short = "y")] - pub yes: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// All core commands that are provided by default. -/// -/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From -/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of -/// `Run` are exported as main executable parameters. -#[derive(Debug, Clone, StructOpt)] -pub enum Subcommand { - /// Build a spec.json file, outputing to stdout. - BuildSpec(BuildSpecCmd), - - /// Export blocks to a file. - ExportBlocks(ExportBlocksCmd), - - /// Import blocks from file. - ImportBlocks(ImportBlocksCmd), - - /// Validte a single block. - CheckBlock(CheckBlockCmd), - - /// Revert chain to the previous state. - Revert(RevertCmd), - - /// Remove the whole chain data. - PurgeChain(PurgeChainCmd), -} - -impl Subcommand { - /// Get the shared parameters of a `CoreParams` command - pub fn get_shared_params(&self) -> &SharedParams { - use Subcommand::*; - - match self { - BuildSpec(params) => ¶ms.shared_params, - ExportBlocks(params) => ¶ms.shared_params, - ImportBlocks(params) => ¶ms.shared_params, - CheckBlock(params) => ¶ms.shared_params, - Revert(params) => ¶ms.shared_params, - PurgeChain(params) => ¶ms.shared_params, - } - } - - /// Run any `CoreParams` command - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - match self { - Subcommand::BuildSpec(cmd) => cmd.run(config), - Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), - Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), - Subcommand::CheckBlock(cmd) => cmd.run(config, builder), - Subcommand::PurgeChain(cmd) => cmd.run(config), - Subcommand::Revert(cmd) => cmd.run(config, builder), - } - } -} - -impl RunCmd { - /// Run the command that runs the node - pub fn run( - self, - mut config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::update_config_for_running_node(&mut config, self)?; - - crate::run_node(config, new_light, new_full, &version) - } -} - -impl BuildSpecCmd { - /// Run the build-spec command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - info!("Building chain spec"); - let mut spec = config.expect_chain_spec().clone(); - let raw_output = self.raw; - - if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { - let node_key = node_key_config( - self.node_key_params.clone(), - &Some(config - .in_chain_config_dir(crate::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a base_path")), - )?; - let keys = node_key.into_keypair()?; - let peer_id = keys.public().into_peer_id(); - let addr = build_multiaddr![ - Ip4([127, 0, 0, 1]), - Tcp(30333u16), - P2p(peer_id) - ]; - spec.add_boot_node(addr) - } - - let json = sc_service::chain_ops::build_spec(spec, raw_output)?; - - print!("{}", json); - - Ok(()) - } -} - -impl ExportBlocksCmd { - /// Run the export-blocks command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { - info!("DB path: {}", path.display()); - } - let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); - let to = self.to.as_ref().and_then(|t| t.parse().ok()); - - let json = self.json; - - let file: Box = match &self.output { - Some(filename) => Box::new(fs::File::create(filename)?), - None => Box::new(io::stdout()), - }; - - run_until_exit(config, |config| { - Ok(builder(config)?.export_blocks(file, from.into(), to, json)) - }) - } -} - -/// Internal trait used to cast to a dynamic type that implements Read and Seek. -trait ReadPlusSeek: Read + Seek {} - -impl ReadPlusSeek for T {} - -impl ImportBlocksCmd { - /// Run the import-blocks command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - crate::fill_import_params( - &mut config, - &self.import_params, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - - let file: Box = match &self.input { - Some(filename) => Box::new(fs::File::open(filename)?), - None => { - let mut buffer = Vec::new(); - io::stdin().read_to_end(&mut buffer)?; - Box::new(io::Cursor::new(buffer)) - }, - }; - - run_until_exit(config, |config| { - Ok(builder(config)?.import_blocks(file, false)) - }) - } -} - -impl CheckBlockCmd { - /// Run the check-block command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_import_params( - &mut config, - &self.import_params, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - crate::fill_config_keystore_in_memory(&mut config)?; - - let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; - let block_id = match FromStr::from_str(input) { - Ok(hash) => BlockId::hash(hash), - Err(_) => match self.input.parse::() { - Ok(n) => BlockId::number((n as u32).into()), - Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), - } - }; - - let start = std::time::Instant::now(); - run_until_exit(config, |config| { - Ok(builder(config)?.check_block(block_id)) - })?; - println!("Completed in {} ms.", start.elapsed().as_millis()); - - Ok(()) - } -} - -impl PurgeChainCmd { - /// Run the purge command - pub fn run( - self, - mut config: Configuration, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - let db_path = match config.expect_database() { - DatabaseConfig::Path { path, .. } => path, - _ => { - eprintln!("Cannot purge custom database implementation"); - return Ok(()); - } - }; - - if !self.yes { - print!("Are you sure to remove {:?}? [y/N]: ", &db_path); - io::stdout().flush().expect("failed to flush stdout"); - - let mut input = String::new(); - io::stdin().read_line(&mut input)?; - let input = input.trim(); - - match input.chars().nth(0) { - Some('y') | Some('Y') => {}, - _ => { - println!("Aborted"); - return Ok(()); - }, - } - } - - match fs::remove_dir_all(&db_path) { - Ok(_) => { - println!("{:?} removed.", &db_path); - Ok(()) - }, - Err(ref err) if err.kind() == io::ErrorKind::NotFound => { - eprintln!("{:?} did not exist.", &db_path); - Ok(()) - }, - Err(err) => Result::Err(err.into()) - } - } -} - -impl RevertCmd { - /// Run the revert command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - let blocks = self.num.parse()?; - builder(config)?.revert_chain(blocks)?; - - Ok(()) - } -} diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..36c62bc979c5c6c0b7f078e124b85f573ca76be5 --- /dev/null +++ b/client/cli/src/params/import_params.rs @@ -0,0 +1,194 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis, config::DatabaseConfig}; + +use crate::error; +use crate::arg_enums::{ + WasmExecutionMethod, TracingReceiver, ExecutionStrategy, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, + DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, + DEFAULT_EXECUTION_SYNCING +}; +use crate::params::PruningParams; + +/// Parameters for block import. +#[derive(Debug, StructOpt, Clone)] +pub struct ImportParams { + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, + + /// Force start with unsafe pruning settings. + /// + /// When running as a validator it is highly recommended to disable state + /// pruning (i.e. 'archive') which is the default. The node will refuse to + /// start as a validator if pruning is enabled unless this option is set. + #[structopt(long = "unsafe-pruning")] + pub unsafe_pruning: bool, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub execution_strategies: ExecutionStrategies, + + /// Limit the memory the database cache can use. + #[structopt(long = "db-cache", value_name = "MiB", default_value = "1024")] + pub database_cache_size: u32, + + /// Specify the state cache size. + #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] + pub state_cache_size: usize, + + /// Comma separated list of targets for tracing + #[structopt(long = "tracing-targets", value_name = "TARGETS")] + pub tracing_targets: Option, + + /// Receiver to process tracing messages + #[structopt( + long = "tracing-receiver", + value_name = "RECEIVER", + possible_values = &TracingReceiver::variants(), + case_insensitive = true, + default_value = "Log" + )] + pub tracing_receiver: TracingReceiver, +} + +impl ImportParams { + /// Put block import CLI params into `config` object. + pub fn update_config( + &self, + mut config: &mut Configuration, + role: sc_service::Roles, + is_dev: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + use sc_client_api::execution_extensions::ExecutionStrategies; + + if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { + *cache_size = Some(self.database_cache_size); + } + + config.state_cache_size = self.state_cache_size; + + self.pruning_params.update_config(&mut config, role, self.unsafe_pruning)?; + + config.wasm_method = self.wasm_method.into(); + + let exec = &self.execution_strategies; + let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { + exec.execution.unwrap_or(if strat == default && is_dev { + ExecutionStrategy::Native + } else { + strat + }).into() + }; + + config.execution_strategies = ExecutionStrategies { + syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), + importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), + block_construction: + exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION), + offchain_worker: + exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), + other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), + }; + + Ok(()) + } +} + +/// Execution strategies parameters. +#[derive(Debug, StructOpt, Clone)] +pub struct ExecutionStrategies { + /// The means of execution used when calling into the runtime while syncing blocks. + #[structopt( + long = "execution-syncing", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_SYNCING.as_str(), + )] + pub execution_syncing: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while importing blocks. + #[structopt( + long = "execution-import-block", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_IMPORT_BLOCK.as_str(), + )] + pub execution_import_block: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while constructing blocks. + #[structopt( + long = "execution-block-construction", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_BLOCK_CONSTRUCTION.as_str(), + )] + pub execution_block_construction: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while using an off-chain worker. + #[structopt( + long = "execution-offchain-worker", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_OFFCHAIN_WORKER.as_str(), + )] + pub execution_offchain_worker: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks. + #[structopt( + long = "execution-other", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_OTHER.as_str(), + )] + pub execution_other: ExecutionStrategy, + + /// The execution strategy that should be used by all execution contexts. + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + conflicts_with_all = &[ + "execution-other", + "execution-offchain-worker", + "execution-block-construction", + "execution-import-block", + "execution-syncing", + ] + )] + pub execution: Option, +} diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..f684cab336423159c6a18ff492c3d7741c4cfded --- /dev/null +++ b/client/cli/src/params/mod.rs @@ -0,0 +1,67 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +mod import_params; +mod transaction_pool_params; +mod shared_params; +mod node_key_params; +mod network_configuration_params; +mod pruning_params; + +use std::str::FromStr; +use std::fmt::Debug; + +pub use crate::params::import_params::*; +pub use crate::params::transaction_pool_params::*; +pub use crate::params::shared_params::*; +pub use crate::params::node_key_params::*; +pub use crate::params::network_configuration_params::*; +pub use crate::params::pruning_params::*; + +/// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. +#[derive(Debug, Clone)] +pub struct BlockNumber(String); + +impl FromStr for BlockNumber { + type Err = String; + + fn from_str(block_number: &str) -> Result { + if block_number.chars().any(|d| !d.is_digit(10)) { + Err(format!( + "Invalid block number: {}, expected decimal formatted unsigned integer", + block_number, + )) + } else { + Ok(Self(block_number.to_owned())) + } + } +} + +impl BlockNumber { + /// Wrapper on top of `std::str::parse` but with `Error` as a `String` + /// + /// See `https://doc.rust-lang.org/std/primitive.str.html#method.parse` for more elaborate + /// documentation. + pub fn parse(&self) -> Result + where + N: FromStr, + N::Err: std::fmt::Debug, + { + self.0 + .parse() + .map_err(|e| format!("BlockNumber: {} parsing failed because of {:?}", self.0, e)) + } +} diff --git a/client/cli/src/params/network_configuration_params.rs b/client/cli/src/params/network_configuration_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..eef679d6a615fa4acc9c33598c0cc2c1a0c04d03 --- /dev/null +++ b/client/cli/src/params/network_configuration_params.rs @@ -0,0 +1,160 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use std::iter; +use std::net::Ipv4Addr; +use structopt::StructOpt; +use sc_network::{ + config::{NonReservedPeerMode, TransportConfig}, multiaddr::Protocol, +}; +use sc_service::{Configuration, RuntimeGenesis}; + +use crate::error; +use crate::params::node_key_params::NodeKeyParams; + +/// Parameters used to create the network configuration. +#[derive(Debug, StructOpt, Clone)] +pub struct NetworkConfigurationParams { + /// Specify a list of bootnodes. + #[structopt(long = "bootnodes", value_name = "URL")] + pub bootnodes: Vec, + + /// Specify a list of reserved node addresses. + #[structopt(long = "reserved-nodes", value_name = "URL")] + pub reserved_nodes: Vec, + + /// Whether to only allow connections to/from reserved nodes. + /// + /// If you are a validator your node might still connect to other validator + /// nodes regardless of whether they are defined as reserved nodes. + #[structopt(long = "reserved-only")] + pub reserved_only: bool, + + /// Specify a list of sentry node public addresses. + #[structopt( + long = "sentry-nodes", + value_name = "URL", + conflicts_with_all = &[ "sentry" ] + )] + pub sentry_nodes: Vec, + + /// Listen on this multiaddress. + #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] + pub listen_addr: Vec, + + /// Specify p2p protocol TCP port. + /// + /// Only used if --listen-addr is not specified. + #[structopt(long = "port", value_name = "PORT")] + pub port: Option, + + /// Forbid connecting to private IPv4 addresses (as specified in + /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with + /// `--reserved-nodes` or `--bootnodes`. + #[structopt(long = "no-private-ipv4")] + pub no_private_ipv4: bool, + + /// Specify the number of outgoing connections we're trying to maintain. + #[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")] + pub out_peers: u32, + + /// Specify the maximum number of incoming connections we're accepting. + #[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")] + pub in_peers: u32, + + /// Disable mDNS discovery. + /// + /// By default, the network will use mDNS to discover other nodes on the + /// local network. This disables it. Automatically implied when using --dev. + #[structopt(long = "no-mdns")] + pub no_mdns: bool, + + /// Maximum number of peers to ask the same blocks in parallel. + /// + /// This allows downlading announced blocks from multiple peers. Decrease to save + /// traffic and risk increased latency. + #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] + pub max_parallel_downloads: u32, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub node_key_params: NodeKeyParams, + + /// Experimental feature flag. + #[structopt(long = "use-yamux-flow-control")] + pub use_yamux_flow_control: bool, +} + +impl NetworkConfigurationParams { + /// Fill the given `NetworkConfiguration` by looking at the cli parameters. + pub fn update_config( + &self, + mut config: &mut Configuration, + config_path: PathBuf, + client_id: String, + is_dev: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + config.network.boot_nodes.extend(self.bootnodes.clone()); + config.network.config_path = Some(config_path.clone()); + config.network.net_config_path = Some(config_path.clone()); + + config.network.reserved_nodes.extend(self.reserved_nodes.clone()); + if self.reserved_only { + config.network.non_reserved_mode = NonReservedPeerMode::Deny; + } + + config.network.sentry_nodes.extend(self.sentry_nodes.clone()); + + for addr in self.listen_addr.iter() { + let addr = addr.parse().ok().ok_or(error::Error::InvalidListenMultiaddress)?; + config.network.listen_addresses.push(addr); + } + + if config.network.listen_addresses.is_empty() { + let port = match self.port { + Some(port) => port, + None => 30333, + }; + + config.network.listen_addresses = vec![ + iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) + .chain(iter::once(Protocol::Tcp(port))) + .collect() + ]; + } + + config.network.client_version = client_id; + self.node_key_params.update_config(&mut config, Some(&config_path))?; + + config.network.in_peers = self.in_peers; + config.network.out_peers = self.out_peers; + + config.network.transport = TransportConfig::Normal { + enable_mdns: !is_dev && !self.no_mdns, + allow_private_ipv4: !self.no_private_ipv4, + wasm_external_transport: None, + use_yamux_flow_control: self.use_yamux_flow_control, + }; + + config.network.max_parallel_downloads = self.max_parallel_downloads; + + Ok(()) + } +} diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..ddc1d6cc21af7ce91715aa64e4b3cbc5bff06722 --- /dev/null +++ b/client/cli/src/params/node_key_params.rs @@ -0,0 +1,244 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{path::PathBuf, str::FromStr}; +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis}; +use sc_network::config::NodeKeyConfig; +use sp_core::H256; + +use crate::error; +use crate::arg_enums::NodeKeyType; + +/// The file name of the node's Ed25519 secret key inside the chain-specific +/// network config directory, if neither `--node-key` nor `--node-key-file` +/// is specified in combination with `--node-key-type=ed25519`. +const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; + +/// Parameters used to create the `NodeKeyConfig`, which determines the keypair +/// used for libp2p networking. +#[derive(Debug, StructOpt, Clone)] +pub struct NodeKeyParams { + /// The secret key to use for libp2p networking. + /// + /// The value is a string that is parsed according to the choice of + /// `--node-key-type` as follows: + /// + /// `ed25519`: + /// The value is parsed as a hex-encoded Ed25519 32 bytes secret key, + /// i.e. 64 hex characters. + /// + /// The value of this option takes precedence over `--node-key-file`. + /// + /// WARNING: Secrets provided as command-line arguments are easily exposed. + /// Use of this option should be limited to development and testing. To use + /// an externally managed secret key, use `--node-key-file` instead. + #[structopt(long = "node-key", value_name = "KEY")] + pub node_key: Option, + + /// The type of secret key to use for libp2p networking. + /// + /// The secret key of the node is obtained as follows: + /// + /// * If the `--node-key` option is given, the value is parsed as a secret key + /// according to the type. See the documentation for `--node-key`. + /// + /// * If the `--node-key-file` option is given, the secret key is read from the + /// specified file. See the documentation for `--node-key-file`. + /// + /// * Otherwise, the secret key is read from a file with a predetermined, + /// type-specific name from the chain-specific network config directory + /// inside the base directory specified by `--base-dir`. If this file does + /// not exist, it is created with a newly generated secret key of the + /// chosen type. + /// + /// The node's secret key determines the corresponding public key and hence the + /// node's peer ID in the context of libp2p. + #[structopt( + long = "node-key-type", + value_name = "TYPE", + possible_values = &NodeKeyType::variants(), + case_insensitive = true, + default_value = "Ed25519" + )] + pub node_key_type: NodeKeyType, + + /// The file from which to read the node's secret key to use for libp2p networking. + /// + /// The contents of the file are parsed according to the choice of `--node-key-type` + /// as follows: + /// + /// `ed25519`: + /// The file must contain an unencoded 32 bytes Ed25519 secret key. + /// + /// If the file does not exist, it is created with a newly generated secret key of + /// the chosen type. + #[structopt(long = "node-key-file", value_name = "FILE")] + pub node_key_file: Option, +} + +impl NodeKeyParams { + /// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context + /// of an optional network config storage directory. + pub fn update_config<'a, G, E>( + &self, + mut config: &'a mut Configuration, + net_config_path: Option<&PathBuf>, + ) -> error::Result<&'a NodeKeyConfig> + where + G: RuntimeGenesis, + { + config.network.node_key = match self.node_key_type { + NodeKeyType::Ed25519 => { + let secret = if let Some(node_key) = self.node_key.as_ref() { + parse_ed25519_secret(node_key)? + } else { + let path = self.node_key_file.clone() + .or_else(|| net_config_path.map(|d| d.join(NODE_KEY_ED25519_FILE))); + + if let Some(path) = path { + sc_network::config::Secret::File(path) + } else { + sc_network::config::Secret::New + } + }; + + NodeKeyConfig::Ed25519(secret) + } + }; + + Ok(&config.network.node_key) + } +} + +/// Create an error caused by an invalid node key argument. +fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { + error::Error::Input(format!("Invalid node key: {}", e)) +} + +/// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. +fn parse_ed25519_secret(hex: &str) -> error::Result { + H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| + sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) + .map(sc_network::config::Secret::Input) + .map_err(invalid_node_key)) +} + +#[cfg(test)] +mod tests { + use sc_network::config::identity::ed25519; + use super::*; + + #[test] + fn test_node_key_config_input() { + fn secret_input(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + NodeKeyType::variants().iter().try_for_each(|t| { + let mut config = Configuration::<(), ()>::default(); + let node_key_type = NodeKeyType::from_str(t).unwrap(); + let sk = match node_key_type { + NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() + }; + let params = NodeKeyParams { + node_key_type, + node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), + node_key_file: None + }; + params.update_config(&mut config, net_config_dir).and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) + if node_key_type == NodeKeyType::Ed25519 && + &sk[..] == ski.as_ref() => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(secret_input(None).is_ok()); + assert!(secret_input(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + } + + #[test] + fn test_node_key_config_file() { + fn secret_file(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + NodeKeyType::variants().iter().try_for_each(|t| { + let mut config = Configuration::<(), ()>::default(); + let node_key_type = NodeKeyType::from_str(t).unwrap(); + let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; + let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); + let params = NodeKeyParams { + node_key_type, + node_key: None, + node_key_file: Some(file.clone()) + }; + params.update_config(&mut config, net_config_dir).and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) + if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(secret_file(None).is_ok()); + assert!(secret_file(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + } + + #[test] + fn test_node_key_config_default() { + fn with_def_params(f: F) -> error::Result<()> + where + F: Fn(NodeKeyParams) -> error::Result<()> + { + NodeKeyType::variants().iter().try_for_each(|t| { + let node_key_type = NodeKeyType::from_str(t).unwrap(); + f(NodeKeyParams { + node_key_type, + node_key: None, + node_key_file: None + }) + }) + } + + fn no_config_dir() -> error::Result<()> { + with_def_params(|params| { + let mut config = Configuration::<(), ()>::default(); + let typ = params.node_key_type; + params.update_config(&mut config, None) + .and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::New) + if typ == NodeKeyType::Ed25519 => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + fn some_config_dir(net_config_dir: &PathBuf) -> error::Result<()> { + with_def_params(|params| { + let mut config = Configuration::<(), ()>::default(); + let dir = PathBuf::from(net_config_dir.clone()); + let typ = params.node_key_type; + params.update_config(&mut config, Some(net_config_dir)) + .and_then(move |c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) + if typ == NodeKeyType::Ed25519 && + f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(no_config_dir().is_ok()); + assert!(some_config_dir(&PathBuf::from_str("x").unwrap()).is_ok()); + } +} diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..ad64d757dcb61acb3fb6928caebfff9c8c778015 --- /dev/null +++ b/client/cli/src/params/pruning_params.rs @@ -0,0 +1,69 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis, PruningMode}; + +use crate::error; + +/// Parameters to define the pruning mode +#[derive(Debug, StructOpt, Clone)] +pub struct PruningParams { + /// Specify the state pruning mode, a number of blocks to keep or 'archive'. + /// + /// Default is to keep all block states if the node is running as a + /// validator (i.e. 'archive'), otherwise state is only kept for the last + /// 256 blocks. + #[structopt(long = "pruning", value_name = "PRUNING_MODE")] + pub pruning: Option, +} + +impl PruningParams { + /// Put block pruning CLI params into `config` object. + pub fn update_config( + &self, + mut config: &mut Configuration, + role: sc_service::Roles, + unsafe_pruning: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + // by default we disable pruning if the node is an authority (i.e. + // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the + // node is an authority and pruning is enabled explicitly, then we error + // unless `unsafe_pruning` is set. + config.pruning = match &self.pruning { + Some(ref s) if s == "archive" => PruningMode::ArchiveAll, + None if role == sc_service::Roles::AUTHORITY => PruningMode::ArchiveAll, + None => PruningMode::default(), + Some(s) => { + if role == sc_service::Roles::AUTHORITY && !unsafe_pruning { + return Err(error::Error::Input( + "Validators should run with state pruning disabled (i.e. archive). \ + You can ignore this check with `--unsafe-pruning`.".to_string() + )); + } + + PruningMode::keep_blocks(s.parse() + .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? + ) + }, + }; + + Ok(()) + } +} diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..03f44796460a28b138fdf9f3a91754a3b685c354 --- /dev/null +++ b/client/cli/src/params/shared_params.rs @@ -0,0 +1,116 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use structopt::StructOpt; +use app_dirs::{AppInfo, AppDataType}; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, + config::DatabaseConfig, ChainSpec, +}; + +use crate::VersionInfo; +use crate::error; + +/// default sub directory to store database +const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; + +/// Shared parameters used by all `CoreParams`. +#[derive(Debug, StructOpt, Clone)] +pub struct SharedParams { + /// Specify the chain specification (one of dev, local or staging). + #[structopt(long = "chain", value_name = "CHAIN_SPEC")] + pub chain: Option, + + /// Specify the development chain. + #[structopt(long = "dev")] + pub dev: bool, + + /// Specify custom base path. + #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] + pub base_path: Option, + + /// Sets a custom logging filter. + #[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")] + pub log: Option, +} + +impl SharedParams { + /// Load spec to `Configuration` from `SharedParams` and spec factory. + pub fn update_config<'a, G, E, F>( + &self, + mut config: &'a mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<&'a ChainSpec> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + let chain_key = match self.chain { + Some(ref chain) => chain.clone(), + None => if self.dev { "dev".into() } else { "".into() } + }; + let spec = match spec_factory(&chain_key)? { + Some(spec) => spec, + None => ChainSpec::from_json_file(PathBuf::from(chain_key))? + }; + + config.network.boot_nodes = spec.boot_nodes().to_vec(); + config.telemetry_endpoints = spec.telemetry_endpoints().clone(); + + config.chain_spec = Some(spec); + + if config.config_dir.is_none() { + config.config_dir = Some(base_path(self, version)); + } + + if config.database.is_none() { + config.database = Some(DatabaseConfig::Path { + path: config + .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) + .expect("We provided a base_path/config_dir."), + cache_size: None, + }); + } + + Ok(config.chain_spec.as_ref().unwrap()) + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + crate::init(self.log.as_ref().map(|v| v.as_ref()).unwrap_or(""), version) + } +} + +fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { + cli.base_path.clone() + .unwrap_or_else(|| + app_dirs::get_app_root( + AppDataType::UserData, + &AppInfo { + name: version.executable_name, + author: version.author + } + ).expect("app directories exist on all supported platforms; qed") + ) +} diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..80c591d1d2dcf2454d5aa059c0c7098744429b0c --- /dev/null +++ b/client/cli/src/params/transaction_pool_params.rs @@ -0,0 +1,49 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::Configuration; +use crate::error; + +/// Parameters used to create the pool configuration. +#[derive(Debug, StructOpt, Clone)] +pub struct TransactionPoolParams { + /// Maximum number of transactions in the transaction pool. + #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] + pub pool_limit: usize, + /// Maximum number of kilobytes of all transactions stored in the pool. + #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] + pub pool_kbytes: usize, +} + +impl TransactionPoolParams { + /// Fill the given `PoolConfiguration` by looking at the cli parameters. + pub fn update_config( + &self, + config: &mut Configuration, + ) -> error::Result<()> { + // ready queue + config.transaction_pool.ready.count = self.pool_limit; + config.transaction_pool.ready.total_bytes = self.pool_kbytes * 1024; + + // future queue + let factor = 10; + config.transaction_pool.future.count = self.pool_limit / factor; + config.transaction_pool.future.total_bytes = self.pool_kbytes * 1024 / factor; + + Ok(()) + } +} diff --git a/client/cli/src/runtime.rs b/client/cli/src/runtime.rs index 62a2245c9e1740500d8f1806bd411a2040d8baad..eccf240f20ff3190cfb5f58809a004a2fadc03e6 100644 --- a/client/cli/src/runtime.rs +++ b/client/cli/src/runtime.rs @@ -21,7 +21,6 @@ use futures::select; use futures::pin_mut; use sc_service::{AbstractService, Configuration}; use crate::error; -use crate::informant; #[cfg(target_family = "unix")] async fn main(func: F) -> Result<(), Box> @@ -124,17 +123,19 @@ where let service = service_builder(config)?; - let informant_future = informant::build(&service); + let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); let _informant_handle = runtime.spawn(informant_future); // we eagerly drop the service so that the internal exit future is fired, // but we need to keep holding a reference to the global telemetry guard + // and drop the runtime first. let _telemetry = service.telemetry(); let f = service.fuse(); pin_mut!(f); runtime.block_on(main(f)).map_err(|e| e.to_string())?; + drop(runtime); Ok(()) } diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index 142b10b9daf2d15ddbf01be434aa1758862b6034..420b0e30053db8392b6cff3f69c7339f4857dba4 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -1,43 +1,45 @@ [package] name = "sc-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Aura consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8", path = "../../../primitives/consensus/aura" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sc-client = { version = "0.8", path = "../../" } -sc-client-api = { version = "2.0.0", path = "../../api" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/aura" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../../primitives/block-builder" } +sc-client = { version = "0.8.0-alpha.1", path = "../../" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" futures = "0.3.1" -futures-timer = "0.4.0" -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sc-keystore = { version = "2.0.0", path = "../../keystore" } +futures-timer = "3.0.1" +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../keystore" } log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-version = { version = "2.0.0", path = "../../../primitives/version" } -sc-consensus-slots = { version = "0.8", path = "../slots" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } +sp-version = { version = "2.0.0-alpha.1", path = "../../../primitives/version" } +sc-consensus-slots = { version = "0.8.0-alpha.1", path = "../slots" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-alpha.1", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../../telemetry" } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8", path = "../../executor" } -sc-network = { version = "0.8", path = "../../network" } -sc-network-test = { version = "0.8.0", path = "../../network/test" } -sc-service = { version = "0.8", path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-alpha.1", path = "../../executor" } +sc-network = { version = "0.8.0-alpha.1", path = "../../network" } +sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } +sc-service = { version = "0.8.0-alpha.1", path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } tokio = "0.1.22" env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/aura/src/digest.rs b/client/consensus/aura/src/digests.rs similarity index 100% rename from client/consensus/aura/src/digest.rs rename to client/consensus/aura/src/digests.rs diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 434314a85353e7f44e662d45e30077f36eb51aa6..355c058645097ca2b53679364803c4e284ec514f 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -79,33 +79,23 @@ pub use sp_consensus_aura::{ }, }; pub use sp_consensus::SyncOracle; -pub use digest::CompatibleDigestItem; +pub use digests::CompatibleDigestItem; -mod digest; +mod digests; type AuthorityId

=

::Public; -/// A slot duration. Create with `get_or_compute`. -#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq)] -pub struct SlotDuration(sc_consensus_slots::SlotDuration); - -impl SlotDuration { - /// Either fetch the slot duration from disk or compute it from the genesis - /// state. - pub fn get_or_compute(client: &C) -> CResult - where - A: Codec, - B: BlockT, - C: AuxStore + ProvideRuntimeApi, - C::Api: AuraApi, - { - sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)).map(Self) - } +/// Slot duration type for Aura. +pub type SlotDuration = sc_consensus_slots::SlotDuration; - /// Get the slot duration in milliseconds. - pub fn get(&self) -> u64 { - self.0.get() - } +/// Get type of `SlotDuration` for Aura. +pub fn slot_duration(client: &C) -> CResult where + A: Codec, + B: BlockT, + C: AuxStore + ProvideRuntimeApi, + C::Api: AuraApi, +{ + SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)) } /// Get slot author for given block along with authorities. @@ -179,10 +169,10 @@ pub fn start_aura( }; register_aura_inherent_data_provider( &inherent_data_providers, - slot_duration.0.slot_duration() + slot_duration.slot_duration() )?; Ok(sc_consensus_slots::start_slot_worker::<_, _, _, _, _, AuraSlotCompatible, _>( - slot_duration.0, + slot_duration, select_chain, worker, sync_oracle, @@ -282,20 +272,13 @@ impl sc_consensus_slots::SimpleSlotWorker for AuraW let signature = pair.sign(header_hash.as_ref()); let signature_digest_item = as CompatibleDigestItem

>::aura_seal(signature); - BlockImportParams { - origin: BlockOrigin::Own, - header, - justification: None, - post_digests: vec![signature_digest_item], - body: Some(body), - storage_changes: Some(storage_changes), - finalized: false, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - } + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(signature_digest_item); + import_block.body = Some(body); + import_block.storage_changes = Some(storage_changes); + import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain); + + import_block }) } @@ -422,21 +405,17 @@ fn find_pre_digest(header: &B::Header) -> Result( +fn check_header( client: &C, slot_now: u64, mut header: B::Header, hash: B::Hash, authorities: &[AuthorityId

], - _transaction_pool: Option<&T>, ) -> Result)>, Error> where DigestItemFor: CompatibleDigestItem

, P::Signature: Decode, C: sc_client_api::backend::AuxStore, P::Public: Encode + Decode + PartialEq + Clone, - T: Send + Sync + 'static, { let seal = match header.digest_mut().pop() { Some(x) => x, @@ -486,14 +465,13 @@ fn check_header( } /// A verifier for Aura blocks. -pub struct AuraVerifier { +pub struct AuraVerifier { client: Arc, phantom: PhantomData

, inherent_data_providers: sp_inherents::InherentDataProviders, - transaction_pool: Option>, } -impl AuraVerifier +impl AuraVerifier where P: Send + Sync + 'static { fn check_inherents( @@ -548,7 +526,7 @@ impl AuraVerifier } #[forbid(deprecated)] -impl Verifier for AuraVerifier where +impl Verifier for AuraVerifier where C: ProvideRuntimeApi + Send + Sync + @@ -560,7 +538,6 @@ impl Verifier for AuraVerifier where P: Pair + Send + Sync + 'static, P::Public: Send + Sync + Hash + Eq + Clone + Decode + Encode + Debug + 'static, P::Signature: Encode + Decode, - T: Send + Sync + 'static, { fn verify( &mut self, @@ -582,13 +559,12 @@ impl Verifier for AuraVerifier where // we add one to allow for some small drift. // FIXME #1019 in the future, alter this queue to allow deferring of // headers - let checked_header = check_header::( + let checked_header = check_header::( &self.client, slot_now + 1, header, hash, &authorities[..], - self.transaction_pool.as_ref().map(|x| &**x), ).map_err(|e| e.to_string())?; match checked_header { CheckedHeader::Checked(pre_header, (slot_num, seal)) => { @@ -637,22 +613,14 @@ impl Verifier for AuraVerifier where _ => None, }); - let block_import_params = BlockImportParams { - origin, - header: pre_header, - post_digests: vec![seal], - body, - storage_changes: None, - finalized: false, - justification, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; - - Ok((block_import_params, maybe_keys)) + let mut import_block = BlockImportParams::new(origin, pre_header); + import_block.post_digests.push(seal); + import_block.body = body; + import_block.justification = justification; + import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain); + import_block.post_hash = Some(hash); + + Ok((import_block, maybe_keys)) } CheckedHeader::Deferred(a, b) => { debug!(target: "aura", "Checking {:?} failed; {:?}, {:?}.", hash, a, b); @@ -790,7 +758,7 @@ impl BlockImport for AuraBlockImport, new_cache: HashMap>, ) -> Result { - let hash = block.post_header().hash(); + let hash = block.post_hash(); let slot_number = find_pre_digest::(&block.header) .expect("valid Aura headers must contain a predigest; \ header has been already verified; qed"); @@ -820,14 +788,13 @@ impl BlockImport for AuraBlockImport( +pub fn import_queue( slot_duration: SlotDuration, block_import: I, justification_import: Option>, finality_proof_import: Option>, client: Arc, inherent_data_providers: InherentDataProviders, - transaction_pool: Option>, ) -> Result>, sp_consensus::Error> where B: BlockT, C::Api: BlockBuilderApi + AuraApi> + ApiExt, @@ -837,7 +804,6 @@ pub fn import_queue( P: Pair + Send + Sync + 'static, P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode, P::Signature: Encode + Decode, - T: Send + Sync + 'static, { register_aura_inherent_data_provider(&inherent_data_providers, slot_duration.get())?; initialize_authorities_cache(&*client)?; @@ -846,7 +812,6 @@ pub fn import_queue( client: client.clone(), inherent_data_providers, phantom: PhantomData, - transaction_pool, }; Ok(BasicQueue::new( verifier, @@ -921,12 +886,11 @@ mod tests { const SLOT_DURATION: u64 = 1000; pub struct AuraTestNet { - peers: Vec>, + peers: Vec>, } impl TestNetFactory for AuraTestNet { - type Specialization = DummySpecialization; - type Verifier = AuraVerifier; + type Verifier = AuraVerifier; type PeerData = (); /// Create new test network with peers and given config. @@ -941,8 +905,7 @@ mod tests { { match client { PeersClient::Full(client, _) => { - let slot_duration = SlotDuration::get_or_compute(&*client) - .expect("slot duration available"); + let slot_duration = slot_duration(&*client).expect("slot duration available"); let inherent_data_providers = InherentDataProviders::new(); register_aura_inherent_data_provider( &inherent_data_providers, @@ -953,7 +916,6 @@ mod tests { AuraVerifier { client, inherent_data_providers, - transaction_pool: Default::default(), phantom: Default::default(), } }, @@ -961,15 +923,15 @@ mod tests { } } - fn peer(&mut self, i: usize) -> &mut Peer { + fn peer(&mut self, i: usize) -> &mut Peer { &mut self.peers[i] } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { &self.peers } - fn mut_peers>)>(&mut self, closure: F) { + fn mut_peers>)>(&mut self, closure: F) { closure(&mut self.peers); } } @@ -1010,8 +972,7 @@ mod tests { .for_each(move |_| future::ready(())) ); - let slot_duration = SlotDuration::get_or_compute(&*client) - .expect("slot duration available"); + let slot_duration = slot_duration(&*client).expect("slot duration available"); let inherent_data_providers = InherentDataProviders::new(); register_aura_inherent_data_provider( diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 23bf9cea2841fb96e5e0149fb609a70963116af2..a05723539a595a0ca58b4445362c7c30efc4f410 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -1,38 +1,41 @@ [package] name = "sc-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "BABE consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../../primitives/application-crypto" } num-bigint = "0.2.3" num-rational = "0.2.2" num-traits = "0.2.8" -sp-version = { version = "2.0.0", path = "../../../primitives/version" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } -sc-keystore = { version = "2.0.0", path = "../../keystore" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sc-client = { version = "0.8", path = "../../" } -sc-consensus-epochs = { version = "0.8", path = "../epochs" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-consensus-uncles = { version = "0.8", path = "../uncles" } -sc-consensus-slots = { version = "0.8", path = "../slots" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -fork-tree = { version = "2.0.0", path = "../../../utils/fork-tree" } +serde = { version = "1.0.104", features = ["derive"] } +sp-version = { version = "2.0.0-alpha.1", path = "../../../primitives/version" } +sp-io = { version = "2.0.0-alpha.1", path = "../../../primitives/io" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sp-timestamp = { version = "2.0.0-alpha.1", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../../telemetry" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../keystore" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } +sc-client = { version = "0.8.0-alpha.1", path = "../../" } +sc-consensus-epochs = { version = "0.8.0-alpha.1", path = "../epochs" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../../primitives/block-builder" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sc-consensus-uncles = { version = "0.8.0-alpha.1", path = "../uncles" } +sc-consensus-slots = { version = "0.8.0-alpha.1", path = "../slots" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +fork-tree = { version = "2.0.0-alpha.1", path = "../../../utils/fork-tree" } futures = "0.3.1" -futures-timer = "0.4.0" +futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" schnorrkel = { version = "0.8.5", features = ["preaudit_deprecated"] } @@ -42,13 +45,13 @@ pdqselect = "0.1.0" derive_more = "0.99.2" [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8", path = "../../executor" } -sc-network = { version = "0.8", path = "../../network" } -sc-network-test = { version = "0.8.0", path = "../../network/test" } -sc-service = { version = "0.8", path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -sc-block-builder = { version = "0.8", path = "../../block-builder" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-alpha.1", path = "../../executor" } +sc-network = { version = "0.8.0-alpha.1", path = "../../network" } +sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } +sc-service = { version = "0.8.0-alpha.1", path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +sc-block-builder = { version = "0.8.0-alpha.1", path = "../../block-builder" } tokio = "0.1.22" env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a2d367df5c06a56edcbc40b7ec94056180f95772 --- /dev/null +++ b/client/consensus/babe/rpc/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "sc-consensus-babe-rpc" +version = "0.8.0-alpha.1" +authors = ["Parity Technologies "] +description = "RPC extensions for the BABE consensus algorithm" +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[dependencies] +sc-consensus-babe = { version = "0.8.0-alpha.1", path = "../" } +jsonrpc-core = "14.0.3" +jsonrpc-core-client = "14.0.3" +jsonrpc-derive = "14.0.3" +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../../primitives/consensus/babe" } +serde = { version = "1.0.104", features=["derive"] } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../../primitives/runtime" } +sc-consensus-epochs = { version = "0.8.0-alpha.1", path = "../../epochs" } +futures = "0.3.1" +derive_more = "0.99.2" +sp-api = { version = "2.0.0-alpha.1", path = "../../../../primitives/api" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../../primitives/core" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../../../keystore" } + +[dev-dependencies] +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../test-utils/runtime/client" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../../../primitives/application-crypto" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../../../primitives/keyring" } +tempfile = "3.1.0" diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1ea7e423dc7a90c7b975cc0788284647106bfabf --- /dev/null +++ b/client/consensus/babe/rpc/src/lib.rs @@ -0,0 +1,247 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! RPC api for babe. + +use sc_consensus_babe::{Epoch, authorship, Config}; +use futures::{FutureExt as _, TryFutureExt as _}; +use jsonrpc_core::{ + Error as RpcError, + futures::future as rpc_future, +}; +use jsonrpc_derive::rpc; +use sc_consensus_epochs::{descendent_query, Epoch as EpochT, SharedEpochChanges}; +use sp_consensus_babe::{ + AuthorityId, + BabeApi as BabeRuntimeApi, + digests::PreDigest, +}; +use serde::{Deserialize, Serialize}; +use sc_keystore::KeyStorePtr; +use sp_api::{ProvideRuntimeApi, BlockId}; +use sp_core::crypto::Pair; +use sp_runtime::traits::{Block as BlockT, Header as _}; +use sp_consensus::{SelectChain, Error as ConsensusError}; +use sp_blockchain::{HeaderBackend, HeaderMetadata, Error as BlockChainError}; +use std::{collections::HashMap, fmt, sync::Arc}; + +type FutureResult = Box + Send>; + +/// Provides rpc methods for interacting with Babe. +#[rpc] +pub trait BabeApi { + /// Returns data about which slots (primary or secondary) can be claimed in the current epoch + /// with the keys in the keystore. + #[rpc(name = "babe_epochAuthorship")] + fn epoch_authorship(&self) -> FutureResult>; +} + +/// Implements the BabeRPC trait for interacting with Babe. +/// +/// Uses a background thread to calculate epoch_authorship data. +pub struct BabeRPCHandler { + /// shared reference to the client. + client: Arc, + /// shared reference to EpochChanges + shared_epoch_changes: SharedEpochChanges, + /// shared reference to the Keystore + keystore: KeyStorePtr, + /// config (actually holds the slot duration) + babe_config: Config, + /// The SelectChain strategy + select_chain: SC, +} + +impl BabeRPCHandler { + /// Creates a new instance of the BabeRpc handler. + pub fn new( + client: Arc, + shared_epoch_changes: SharedEpochChanges, + keystore: KeyStorePtr, + babe_config: Config, + select_chain: SC, + ) -> Self { + + Self { + client, + shared_epoch_changes, + keystore, + babe_config, + select_chain, + } + } +} + +impl BabeApi for BabeRPCHandler + where + B: BlockT, + C: ProvideRuntimeApi + HeaderBackend + HeaderMetadata + 'static, + C::Api: BabeRuntimeApi, + ::Error: fmt::Debug, + SC: SelectChain + Clone + 'static, +{ + fn epoch_authorship(&self) -> FutureResult> { + let ( + babe_config, + keystore, + shared_epoch, + client, + select_chain, + ) = ( + self.babe_config.clone(), + self.keystore.clone(), + self.shared_epoch_changes.clone(), + self.client.clone(), + self.select_chain.clone(), + ); + let future = async move { + let header = select_chain.best_chain().map_err(Error::Consensus)?; + let epoch_start = client.runtime_api() + .current_epoch_start(&BlockId::Hash(header.hash())) + .map_err(|err| { + Error::StringError(format!("{:?}", err)) + })?; + let epoch = epoch_data(&shared_epoch, &client, &babe_config, epoch_start, &select_chain)?; + let (epoch_start, epoch_end) = (epoch.start_slot(), epoch.end_slot()); + + let mut claims: HashMap = HashMap::new(); + + for slot_number in epoch_start..epoch_end { + let epoch = epoch_data(&shared_epoch, &client, &babe_config, slot_number, &select_chain)?; + if let Some((claim, key)) = authorship::claim_slot(slot_number, &epoch, &babe_config, &keystore) { + match claim { + PreDigest::Primary { .. } => { + claims.entry(key.public()).or_default().primary.push(slot_number); + } + PreDigest::Secondary { .. } => { + claims.entry(key.public()).or_default().secondary.push(slot_number); + } + }; + } + } + + Ok(claims) + }.boxed(); + + Box::new(future.compat()) + } +} + +/// Holds information about the `slot_number`'s that can be claimed by a given key. +#[derive(Default, Debug, Deserialize, Serialize)] +pub struct EpochAuthorship { + /// the array of primary slots that can be claimed + primary: Vec, + /// the array of secondary slots that can be claimed + secondary: Vec, +} + +/// Errors encountered by the RPC +#[derive(Debug, derive_more::Display, derive_more::From)] +pub enum Error { + /// Consensus error + Consensus(ConsensusError), + /// Errors that can be formatted as a String + StringError(String) +} + +impl From for jsonrpc_core::Error { + fn from(error: Error) -> Self { + jsonrpc_core::Error { + message: format!("{}", error).into(), + code: jsonrpc_core::ErrorCode::ServerError(1234), + data: None, + } + } +} + +/// fetches the epoch data for a given slot_number. +fn epoch_data( + epoch_changes: &SharedEpochChanges, + client: &Arc, + babe_config: &Config, + slot_number: u64, + select_chain: &SC, +) -> Result + where + B: BlockT, + C: HeaderBackend + HeaderMetadata + 'static, + SC: SelectChain, +{ + let parent = select_chain.best_chain()?; + epoch_changes.lock().epoch_for_child_of( + descendent_query(&**client), + &parent.hash(), + parent.number().clone(), + slot_number, + |slot| babe_config.genesis_epoch(slot), + ) + .map_err(|e| Error::Consensus(ConsensusError::ChainLookup(format!("{:?}", e))))? + .map(|e| e.into_inner()) + .ok_or(Error::Consensus(ConsensusError::InvalidAuthoritiesSet)) +} + +#[cfg(test)] +mod tests { + use super::*; + use substrate_test_runtime_client::{ + DefaultTestClientBuilderExt, + TestClientBuilderExt, + TestClientBuilder, + }; + use sp_application_crypto::AppPair; + use sp_keyring::Ed25519Keyring; + use sc_keystore::Store; + + use std::sync::Arc; + use sc_consensus_babe::{Config, block_import, AuthorityPair}; + use jsonrpc_core::IoHandler; + + /// creates keystore backed by a temp file + fn create_temp_keystore(authority: Ed25519Keyring) -> (KeyStorePtr, tempfile::TempDir) { + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = Store::open(keystore_path.path(), None).expect("Creates keystore"); + keystore.write().insert_ephemeral_from_seed::

(&authority.to_seed()) + .expect("Creates authority key"); + + (keystore, keystore_path) + } + + #[test] + fn rpc() { + let builder = TestClientBuilder::new(); + let (client, longest_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); + let config = Config::get_or_compute(&*client).expect("config available"); + let (_, link) = block_import( + config.clone(), + client.clone(), + client.clone(), + ).expect("can initialize block-import"); + + let epoch_changes = link.epoch_changes().clone(); + let select_chain = longest_chain; + let keystore = create_temp_keystore::(Ed25519Keyring::Alice).0; + let handler = BabeRPCHandler::new(client.clone(), epoch_changes, keystore, config, select_chain); + let mut io = IoHandler::new(); + + io.extend_with(BabeApi::to_delegate(handler)); + let request = r#"{"jsonrpc":"2.0","method":"babe_epochAuthorship","params": [],"id":1}"#; + let response = r#"{"jsonrpc":"2.0","result":{"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY":{"primary":[0],"secondary":[1,2,4]}},"id":1}"#; + + assert_eq!(Some(response.into()), io.handle_request_sync(request)); + } +} diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 8b28aefa2f77a1ce276de51b020ea30e8f0f9875..a01ea63bbe111ef193d65de6a4264d7eb8a5ede4 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -86,16 +86,15 @@ pub(super) fn secondary_slot_author( Some(&expected_author.0) } -#[allow(deprecated)] pub(super) fn make_transcript( randomness: &[u8], slot_number: u64, epoch: u64, ) -> Transcript { let mut transcript = Transcript::new(&BABE_ENGINE_ID); - transcript.commit_bytes(b"slot number", &slot_number.to_le_bytes()); - transcript.commit_bytes(b"current epoch", &epoch.to_le_bytes()); - transcript.commit_bytes(b"chain randomness", randomness); + transcript.append_u64(b"slot number", slot_number); + transcript.append_u64(b"current epoch", epoch); + transcript.append_message(b"chain randomness", randomness); transcript } @@ -144,7 +143,7 @@ fn claim_secondary_slot( /// a primary VRF based slot. If we are not able to claim it, then if we have /// secondary slots enabled for the given epoch, we will fallback to trying to /// claim a secondary slot. -pub(super) fn claim_slot( +pub fn claim_slot( slot_number: SlotNumber, epoch: &Epoch, config: &BabeConfiguration, diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index f9e3ef98d67350d5bde63ac736dcd9ec0ab45fa8..000642fec4a26d8a2dde963e6e43c7260f6f8787 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -93,12 +93,9 @@ use sp_consensus_babe::inherents::BabeInherentData; use sp_timestamp::{TimestampInherentData, InherentType as TimestampInherent}; use sp_consensus::import_queue::{Verifier, BasicQueue, CacheKeyId}; use sc_client_api::{ - backend::{AuxStore, Backend}, - call_executor::CallExecutor, + backend::AuxStore, BlockchainEvents, ProvideUncles, }; -use sc_client::Client; - use sp_block_builder::BlockBuilder as BlockBuilderApi; use futures::prelude::*; @@ -119,7 +116,7 @@ use sp_api::ApiExt; mod aux_schema; mod verification; -mod authorship; +pub mod authorship; #[cfg(test)] mod tests; @@ -487,27 +484,16 @@ impl sc_consensus_slots::SimpleSlotWorker for BabeWork let signature = pair.sign(header_hash.as_ref()); let digest_item = as CompatibleDigestItem>::babe_seal(signature); - BlockImportParams { - origin: BlockOrigin::Own, - header, - justification: None, - post_digests: vec![digest_item], - body: Some(body), - storage_changes: Some(storage_changes), - finalized: false, - auxiliary: Vec::new(), // block-weight is written in block import. - intermediates: { - let mut intermediates = HashMap::new(); - intermediates.insert( - Cow::from(INTERMEDIATE_KEY), - Box::new(BabeIntermediate { epoch }) as Box, - ); - intermediates - }, - fork_choice: None, - allow_missing_state: false, - import_existing: false, - } + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(digest_item); + import_block.body = Some(body); + import_block.storage_changes = Some(storage_changes); + import_block.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(BabeIntermediate { epoch }) as Box, + ); + + import_block }) } @@ -666,27 +652,28 @@ impl BabeLink { } /// A verifier for Babe blocks. -pub struct BabeVerifier { - client: Arc>, - api: Arc, +pub struct BabeVerifier { + client: Arc, inherent_data_providers: sp_inherents::InherentDataProviders, config: Config, epoch_changes: SharedEpochChanges, time_source: TimeSource, } -impl BabeVerifier { +impl BabeVerifier + where + Block: BlockT, + Client: HeaderBackend + HeaderMetadata + ProvideRuntimeApi, + Client::Api: BlockBuilderApi, +{ fn check_inherents( &self, block: Block, block_id: BlockId, inherent_data: InherentData, ) -> Result<(), Error> - where - PRA: ProvideRuntimeApi, - PRA::Api: BlockBuilderApi { - let inherent_res = self.api.runtime_api().check_inherents( + let inherent_res = self.client.runtime_api().check_inherents( &block_id, block, inherent_data, @@ -704,57 +691,11 @@ impl BabeVerifier { } } -#[allow(dead_code)] -fn median_algorithm( - median_required_blocks: u64, - slot_duration: u64, - slot_number: u64, - slot_now: u64, - time_source: &mut (Option, Vec<(Instant, u64)>), -) { - let num_timestamps = time_source.1.len(); - if num_timestamps as u64 >= median_required_blocks && median_required_blocks > 0 { - let mut new_list: Vec<_> = time_source.1.iter().map(|&(t, sl)| { - let offset: u128 = u128::from(slot_duration) - .checked_mul(1_000_000u128) // self.config.slot_duration returns milliseconds - .and_then(|x| { - x.checked_mul(u128::from(slot_number).saturating_sub(u128::from(sl))) - }) - .expect("we cannot have timespans long enough for this to overflow; qed"); - - const NANOS_PER_SEC: u32 = 1_000_000_000; - let nanos = (offset % u128::from(NANOS_PER_SEC)) as u32; - let secs = (offset / u128::from(NANOS_PER_SEC)) as u64; - - t + Duration::new(secs, nanos) - }).collect(); - - // Use a partial sort to move the median timestamp to the middle of the list - pdqselect::select(&mut new_list, num_timestamps / 2); - - let &median = new_list - .get(num_timestamps / 2) - .expect("we have at least one timestamp, so this is a valid index; qed"); - - let now = Instant::now(); - if now >= median { - time_source.0.replace(now - median); - } - - time_source.1.clear(); - } else { - time_source.1.push((Instant::now(), slot_now)) - } -} - -impl Verifier for BabeVerifier where +impl Verifier for BabeVerifier where Block: BlockT, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, - PRA: ProvideRuntimeApi + Send + Sync + AuxStore + ProvideCache, - PRA::Api: BlockBuilderApi - + BabeApi, + Client: HeaderMetadata + HeaderBackend + ProvideRuntimeApi + + Send + Sync + AuxStore + ProvideCache, + Client::Api: BlockBuilderApi + BabeApi, { fn verify( &mut self, @@ -823,7 +764,7 @@ impl Verifier for BabeVerifier Verifier for BabeVerifier ?pre_header); - let mut intermediates = HashMap::new(); - intermediates.insert( + let mut import_block = BlockImportParams::new(origin, pre_header); + import_block.post_digests.push(verified_info.seal); + import_block.body = body; + import_block.justification = justification; + import_block.intermediates.insert( Cow::from(INTERMEDIATE_KEY), - Box::new(BabeIntermediate { - epoch, - }) as Box, + Box::new(BabeIntermediate { epoch }) as Box, ); + import_block.post_hash = Some(hash); - let block_import_params = BlockImportParams { - origin, - header: pre_header, - post_digests: vec![verified_info.seal], - body, - storage_changes: None, - finalized: false, - justification, - auxiliary: Vec::new(), - intermediates, - fork_choice: None, - allow_missing_state: false, - import_existing: false, - }; - - Ok((block_import_params, Default::default())) + Ok((import_block, Default::default())) } CheckedHeader::Deferred(a, b) => { debug!(target: "babe", "Checking {:?} failed; {:?}, {:?}.", hash, a, b); @@ -924,37 +852,33 @@ fn register_babe_inherent_data_provider( /// it is missing. /// /// The epoch change tree should be pruned as blocks are finalized. -pub struct BabeBlockImport { +pub struct BabeBlockImport { inner: I, - client: Arc>, - api: Arc, + client: Arc, epoch_changes: SharedEpochChanges, config: Config, } -impl Clone for BabeBlockImport { +impl Clone for BabeBlockImport { fn clone(&self) -> Self { BabeBlockImport { inner: self.inner.clone(), client: self.client.clone(), - api: self.api.clone(), epoch_changes: self.epoch_changes.clone(), config: self.config.clone(), } } } -impl BabeBlockImport { +impl BabeBlockImport { fn new( - client: Arc>, - api: Arc, + client: Arc, epoch_changes: SharedEpochChanges, block_import: I, config: Config, ) -> Self { BabeBlockImport { client, - api, inner: block_import, epoch_changes, config, @@ -962,26 +886,23 @@ impl BabeBlockImport { } } -impl BlockImport for BabeBlockImport where +impl BlockImport for BabeBlockImport where Block: BlockT, - I: BlockImport> + Send + Sync, - I::Error: Into, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - Client: AuxStore, - RA: Send + Sync, - PRA: ProvideRuntimeApi + ProvideCache, - PRA::Api: BabeApi + ApiExt, + Inner: BlockImport> + Send + Sync, + Inner::Error: Into, + Client: HeaderBackend + HeaderMetadata + + AuxStore + ProvideRuntimeApi + ProvideCache + Send + Sync, + Client::Api: BabeApi + ApiExt, { type Error = ConsensusError; - type Transaction = sp_api::TransactionFor; + type Transaction = sp_api::TransactionFor; fn import_block( &mut self, mut block: BlockImportParams, new_cache: HashMap>, ) -> Result { - let hash = block.post_header().hash(); + let hash = block.post_hash(); let number = block.header.number().clone(); // early exit if block already in chain, otherwise the check for @@ -998,7 +919,7 @@ impl BlockImport for BabeBlockImport::ParentUnavailable(parent_hash, hash) @@ -1070,7 +991,7 @@ impl BlockImport for BabeBlockImport BlockImport for BabeBlockImport BlockImport for BabeBlockImport( - client: &Client, +fn prune_finalized( + client: Arc, epoch_changes: &mut EpochChangesFor, ) -> Result<(), ConsensusError> where Block: BlockT, - E: CallExecutor + Send + Sync, - B: Backend, - RA: Send + Sync, + Client: HeaderBackend + HeaderMetadata, { - let info = client.chain_info(); + let info = client.info(); let finalized_slot = { - let finalized_header = client.header(&BlockId::Hash(info.finalized_hash)) + let finalized_header = client.header(BlockId::Hash(info.finalized_hash)) .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e)))? .expect("best finalized hash was given by client; \ finalized headers must exist in db; qed"); @@ -1214,16 +1133,12 @@ fn prune_finalized( /// /// Also returns a link object used to correctly instantiate the import queue /// and background worker. -pub fn block_import( +pub fn block_import( config: Config, wrapped_block_import: I, - client: Arc>, - api: Arc, -) -> ClientResult<(BabeBlockImport, BabeLink)> where - B: Backend, - E: CallExecutor + Send + Sync, - RA: Send + Sync, - Client: AuxStore, + client: Arc, +) -> ClientResult<(BabeBlockImport, BabeLink)> where + Client: AuxStore + HeaderBackend + HeaderMetadata, { let epoch_changes = aux_schema::load_epoch_changes::(&*client)?; let link = BabeLink { @@ -1236,13 +1151,12 @@ pub fn block_import( // epoch tree it is useful as a migration, so that nodes prune long trees on // startup rather than waiting until importing the next epoch change block. prune_finalized( - &client, + client.clone(), &mut epoch_changes.lock(), )?; let import = BabeBlockImport::new( client, - api, epoch_changes, wrapped_block_import, config, @@ -1260,28 +1174,24 @@ pub fn block_import( /// /// The block import object provided must be the `BabeBlockImport` or a wrapper /// of it, otherwise crucial import logic will be omitted. -pub fn import_queue( +pub fn import_queue( babe_link: BabeLink, - block_import: I, + block_import: Inner, justification_import: Option>, finality_proof_import: Option>, - client: Arc>, - api: Arc, + client: Arc, inherent_data_providers: InherentDataProviders, -) -> ClientResult>> where - B: Backend + 'static, - I: BlockImport> +) -> ClientResult>> where + Inner: BlockImport> + Send + Sync + 'static, - E: CallExecutor + Clone + Send + Sync + 'static, - RA: Send + Sync + 'static, - PRA: ProvideRuntimeApi + ProvideCache + Send + Sync + AuxStore + 'static, - PRA::Api: BlockBuilderApi + BabeApi + ApiExt, + Client: ProvideRuntimeApi + ProvideCache + Send + Sync + AuxStore + 'static, + Client: HeaderBackend + HeaderMetadata, + Client::Api: BlockBuilderApi + BabeApi + ApiExt, { register_babe_inherent_data_provider(&inherent_data_providers, babe_link.config.slot_duration)?; let verifier = BabeVerifier { client: client.clone(), - api, inherent_data_providers, config: babe_link.config, epoch_changes: babe_link.epoch_changes, diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 687f23e646f668b0733007ff5322a4bdbeac2cdf..4045e18b5c3fb833874f9d2f003dc008b6788a55 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -199,20 +199,14 @@ impl> BlockImport for PanickingBlockImport< } pub struct BabeTestNet { - peers: Vec, DummySpecialization>>, + peers: Vec>>, } type TestHeader = ::Header; type TestExtrinsic = ::Extrinsic; pub struct TestVerifier { - inner: BabeVerifier< - substrate_test_runtime_client::Backend, - substrate_test_runtime_client::Executor, - TestBlock, - substrate_test_runtime_client::runtime::RuntimeApi, - PeersFullClient, - >, + inner: BabeVerifier, mutator: Mutator, } @@ -229,7 +223,7 @@ impl Verifier for TestVerifier { ) -> Result<(BlockImportParams, Option)>>), String> { // apply post-sealing mutations (i.e. stripping seal, if desired). (self.mutator)(&mut header, Stage::PostSeal); - Ok(self.inner.verify(origin, header, justification, body).expect("verification failed!")) + self.inner.verify(origin, header, justification, body) } } @@ -242,7 +236,6 @@ pub struct PeerData { } impl TestNetFactory for BabeTestNet { - type Specialization = DummySpecialization; type Verifier = TestVerifier; type PeerData = Option; @@ -271,7 +264,6 @@ impl TestNetFactory for BabeTestNet { config, client.clone(), client.clone(), - client.clone(), ).expect("can initialize block-import"); let block_import = PanickingBlockImport(block_import); @@ -305,7 +297,6 @@ impl TestNetFactory for BabeTestNet { TestVerifier { inner: BabeVerifier { client: client.clone(), - api: client, inherent_data_providers: data.inherent_data_providers.clone(), config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), @@ -315,17 +306,17 @@ impl TestNetFactory for BabeTestNet { } } - fn peer(&mut self, i: usize) -> &mut Peer { - trace!(target: "babe", "Retreiving a peer"); + fn peer(&mut self, i: usize) -> &mut Peer { + trace!(target: "babe", "Retrieving a peer"); &mut self.peers[i] } - fn peers(&self) -> &Vec> { - trace!(target: "babe", "Retreiving peers"); + fn peers(&self) -> &Vec> { + trace!(target: "babe", "Retrieving peers"); &self.peers } - fn mut_peers>)>( + fn mut_peers>)>( &mut self, closure: F, ) { @@ -423,7 +414,14 @@ fn run_one_test( } runtime.spawn(futures01::future::poll_fn(move || { - net.lock().poll(); + let mut net = net.lock(); + net.poll(); + for p in net.peers() { + for (h, e) in p.failed_verifications() { + panic!("Verification failed for {:?}: {}", h, e); + } + } + Ok::<_, ()>(futures01::Async::NotReady::<()>) })); @@ -593,30 +591,15 @@ fn propose_and_import_block( h }; - let import_result = block_import.import_block( - BlockImportParams { - origin: BlockOrigin::Own, - header: block.header, - justification: None, - post_digests: vec![seal], - body: Some(block.extrinsics), - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: { - let mut intermediates = HashMap::new(); - intermediates.insert( - Cow::from(INTERMEDIATE_KEY), - Box::new(BabeIntermediate { epoch }) as Box, - ); - intermediates - }, - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }, - Default::default(), - ).unwrap(); + let mut import = BlockImportParams::new(BlockOrigin::Own, block.header); + import.post_digests.push(seal); + import.body = Some(block.extrinsics); + import.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(BabeIntermediate { epoch }) as Box, + ); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + let import_result = block_import.import_block(import, Default::default()).unwrap(); match import_result { ImportResult::Imported(_) => {}, diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index e08553a241d1c41e7c8d175bcaedd3fc03961610..8fc7d8130368af6c4cd67771aa7b421aea63386b 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -1,14 +1,17 @@ [package] name = "sc-consensus-epochs" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Generic epochs-based utilities for consensus" edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } parking_lot = "0.10.0" -fork-tree = { version = "2.0.0", path = "../../../utils/fork-tree" } -sp-runtime = { path = "../../../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sc-client-api = { path = "../../api" } +fork-tree = { version = "2.0.0-alpha.1", path = "../../../utils/fork-tree" } +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.1"} +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sc-client-api = { path = "../../api" , version = "2.0.0-alpha.1"} diff --git a/client/consensus/epochs/src/lib.rs b/client/consensus/epochs/src/lib.rs index cf3d9f5c4c2c2e178789ecd535128d14465c102d..05ee611de137853e32ef4bfc4e22e53f6c17f914 100644 --- a/client/consensus/epochs/src/lib.rs +++ b/client/consensus/epochs/src/lib.rs @@ -141,7 +141,7 @@ impl ViableEpoch where } } -/// The datatype encoded on disk. +/// The data type encoded on disk. #[derive(Clone, Encode, Decode)] pub enum PersistedEpoch { /// Genesis persisted epoch data. epoch_0, epoch_1. diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index 71c1f6e5583a27edb96b6d7710a31112ce6f169d..f432523a43d601a69f27915ace624334cdd4b89e 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "sc-consensus-manual-seal" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Manual sealing engine for Substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" @@ -16,19 +18,19 @@ log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0", features=["derive"] } -sc-client = { path = "../../../client" } -sc-client-api = { path = "../../../client/api" } -sc-transaction-pool = { path = "../../transaction-pool" } -sp-blockchain = { path = "../../../primitives/blockchain" } -sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" } -sp-inherents = { path = "../../../primitives/inherents" } -sp-runtime = { path = "../../../primitives/runtime" } -sp-transaction-pool = { path = "../../../primitives/transaction-pool" } +sc-client = { path = "../../../client" , version = "0.8.0-alpha.1"} +sc-client-api = { path = "../../../client/api" , version = "2.0.0-alpha.1"} +sc-transaction-pool = { path = "../../transaction-pool" , version = "2.0.0-alpha.1"} +sp-blockchain = { path = "../../../primitives/blockchain" , version = "2.0.0-alpha.1"} +sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" , version = "0.8.0-alpha.1"} +sp-inherents = { path = "../../../primitives/inherents" , version = "2.0.0-alpha.1"} +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.1"} +sp-transaction-pool = { path = "../../../primitives/transaction-pool" , version = "2.0.0-alpha.1"} [dev-dependencies] -sc-basic-authorship = { path = "../../basic-authorship" } -substrate-test-runtime-client = { path = "../../../test-utils/runtime/client" } -substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool" } +sc-basic-authorship = { path = "../../basic-authorship" , version = "0.8.0-alpha.1"} +substrate-test-runtime-client = { path = "../../../test-utils/runtime/client" , version = "2.0.0-dev"} +substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool" , version = "2.0.0-dev"} tokio = { version = "0.2", features = ["rt-core", "macros"] } env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 029f746e625e4172bf6f7e8621856602de03afc7..c4336485a14acbe3ac1c6d7b9493a80bd577e5a8 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -89,20 +89,11 @@ impl Verifier for ManualSealVerifier { justification: Option, body: Option>, ) -> Result<(BlockImportParams, Option)>>), String> { - let import_params = BlockImportParams { - origin, - header, - justification, - post_digests: Vec::new(), - body, - storage_changes: None, - finalized: true, - auxiliary: Vec::new(), - intermediates: HashMap::new(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; + let mut import_params = BlockImportParams::new(origin, header); + import_params.justification = justification; + import_params.body = body; + import_params.finalized = true; + import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain); Ok((import_params, None)) } @@ -198,7 +189,7 @@ pub async fn run_instant_seal( { // instant-seal creates blocks as soon as transactions are imported // into the transaction pool. - let seal_block_channel = pool.import_notification_stream() + let seal_block_channel = pool.validated_pool().import_notification_stream() .map(|_| { EngineCommand::SealNewBlock { create_empty: false, @@ -252,7 +243,7 @@ mod tests { let client = Arc::new(builder.build()); let select_chain = LongestChain::new(backend.clone()); let inherent_data_providers = InherentDataProviders::new(); - let pool = Arc::new(BasicPool::new(Options::default(), api())); + let pool = Arc::new(BasicPool::new(Options::default(), api()).0); let env = ProposerFactory { transaction_pool: pool.clone(), client: client.clone(), @@ -260,7 +251,7 @@ mod tests { // this test checks that blocks are created as soon as transactions are imported into the pool. let (sender, receiver) = futures::channel::oneshot::channel(); let mut sender = Arc::new(Some(sender)); - let stream = pool.pool().import_notification_stream() + let stream = pool.pool().validated_pool().import_notification_stream() .map(move |_| { // we're only going to submit one tx so this fn will only be called once. let mut_sender = Arc::get_mut(&mut sender).unwrap(); @@ -317,7 +308,7 @@ mod tests { let client = Arc::new(builder.build()); let select_chain = LongestChain::new(backend.clone()); let inherent_data_providers = InherentDataProviders::new(); - let pool = Arc::new(BasicPool::new(Options::default(), api())); + let pool = Arc::new(BasicPool::new(Options::default(), api()).0); let env = ProposerFactory { transaction_pool: pool.clone(), client: client.clone(), @@ -386,7 +377,7 @@ mod tests { let select_chain = LongestChain::new(backend.clone()); let inherent_data_providers = InherentDataProviders::new(); let pool_api = api(); - let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone())); + let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone()).0); let env = ProposerFactory { transaction_pool: pool.clone(), client: client.clone(), diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_new_block.rs index 53dc82d353e673a82c6159707e2e2bc3c14a712a..39d73e16ab74fdadde2e3a597fa2ef71615cc9fa 100644 --- a/client/consensus/manual-seal/src/seal_new_block.rs +++ b/client/consensus/manual-seal/src/seal_new_block.rs @@ -92,7 +92,7 @@ pub async fn seal_new_block( SC: SelectChain, { let future = async { - if pool.status().ready == 0 && !create_empty { + if pool.validated_pool().status().ready == 0 && !create_empty { return Err(Error::EmptyTransactionPool) } @@ -121,20 +121,10 @@ pub async fn seal_new_block( } let (header, body) = proposal.block.deconstruct(); - let params = BlockImportParams { - origin: BlockOrigin::Own, - header: header.clone(), - justification: None, - post_digests: Vec::new(), - body: Some(body), - finalized: finalize, - storage_changes: None, - auxiliary: Vec::new(), - intermediates: HashMap::new(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; + let mut params = BlockImportParams::new(BlockOrigin::Own, header.clone()); + params.body = Some(body); + params.finalized = finalize; + params.fork_choice = Some(ForkChoiceStrategy::LongestChain); match block_import.import_block(params, HashMap::new())? { ImportResult::Imported(aux) => { diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index b31d9406e1e852101016f1af51a9ef9c9cafaaf4..d3e0be60f4d416f3af5335436b1b012c81f5b829 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "sc-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "PoW consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-consensus-pow = { version = "0.8", path = "../../../primitives/consensus/pow" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } +sp-block-builder = { version = "2.0.0-alpha.1", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } +sp-consensus-pow = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/pow" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } log = "0.4.8" futures = { version = "0.3.1", features = ["compat"] } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } +sp-timestamp = { version = "2.0.0-alpha.1", path = "../../../primitives/timestamp" } derive_more = "0.99.2" diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index d656f71a15b19289172571c5000bfb42659deb66..49d2e64f6050c92b8603e6a3cf034e19263085a1 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -111,9 +111,7 @@ fn aux_key>(hash: &T) -> Vec { /// Intermediate value passed to block importer. #[derive(Encode, Decode, Clone, Debug, Default)] -pub struct PowIntermediate { - /// The header hash with seal, used for auxiliary key. - pub hash: B::Hash, +pub struct PowIntermediate { /// Difficulty of the block, if known. pub difficulty: Option, } @@ -331,7 +329,7 @@ impl BlockImport for PowBlockImport return Err(Error::::HeaderUnsealed(block.header.hash()).into()), }; - let intermediate = block.take_intermediate::>( + let intermediate = block.take_intermediate::>( INTERMEDIATE_KEY )?; @@ -353,7 +351,7 @@ impl BlockImport for PowBlockImport Verifier for PowVerifier where let hash = header.hash(); let (checked_header, seal) = self.check_header(header)?; - let intermediate = PowIntermediate:: { - hash: hash, + let intermediate = PowIntermediate:: { difficulty: None, }; - let import_block = BlockImportParams { - origin, - header: checked_header, - post_digests: vec![seal], - body, - storage_changes: None, - finalized: false, - justification, - intermediates: { - let mut ret = HashMap::new(); - ret.insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box); - ret - }, - auxiliary: vec![], - fork_choice: None, - allow_missing_state: false, - import_existing: false, - }; + let mut import_block = BlockImportParams::new(origin, checked_header); + import_block.post_digests.push(seal); + import_block.body = body; + import_block.justification = justification; + import_block.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(intermediate) as Box + ); + import_block.post_hash = Some(hash); Ok((import_block, None)) } @@ -661,29 +649,19 @@ fn mine_loop( (hash, seal) }; - let intermediate = PowIntermediate:: { - hash, + let intermediate = PowIntermediate:: { difficulty: Some(difficulty), }; - let import_block = BlockImportParams { - origin: BlockOrigin::Own, - header, - justification: None, - post_digests: vec![seal], - body: Some(body), - storage_changes: Some(proposal.storage_changes), - intermediates: { - let mut ret = HashMap::new(); - ret.insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box); - ret - }, - finalized: false, - auxiliary: vec![], - fork_choice: None, - allow_missing_state: false, - import_existing: false, - }; + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(seal); + import_block.body = Some(body); + import_block.storage_changes = Some(proposal.storage_changes); + import_block.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(intermediate) as Box + ); + import_block.post_hash = Some(hash); block_import.import_block(import_block, HashMap::default()) .map_err(|e| Error::BlockBuiltError(best_hash, e))?; diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index d500a0fdc773271b9f27458bde55d68fa32ed8d5..87422a8c6c93a122a25f58d1fd1c1fef50158d8c 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -1,27 +1,29 @@ [package] name = "sc-consensus-slots" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Generic slots-based utilities for consensus" edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0", path = "../../../primitives/state-machine" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../../primitives/state-machine" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../../telemetry" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } futures = "0.3.1" -futures-timer = "2.0" +futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index 8bc2547a49e39ea96a0856f36bc878e421f648ab..892c8b136436b9fa226f2cbd4617c4729870dede 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -287,13 +287,13 @@ pub trait SimpleSlotWorker { info!( "Pre-sealed block for proposal at {}. Hash now {:?}, previously {:?}.", header_num, - block_import_params.post_header().hash(), + block_import_params.post_hash(), header_hash, ); telemetry!(CONSENSUS_INFO; "slots.pre_sealed_block"; "header_num" => ?header_num, - "hash_now" => ?block_import_params.post_header().hash(), + "hash_now" => ?block_import_params.post_hash(), "hash_previously" => ?header_hash, ); diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index f336564c4ec14db9fca99f0e50c2155d81dc35ac..f263cd217101517ee1ec45ae1772d7558eb37794 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "sc-consensus-uncles" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] description = "Generic uncle inclusion utilities for consensus" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-authorship = { version = "2.0.0", path = "../../../primitives/authorship" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-authorship = { version = "2.0.0-alpha.1", path = "../../../primitives/authorship" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../../primitives/inherents" } log = "0.4.8" diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index d35a554d3f29e338bf197667488e035f601a2df2..6446608d35fd1a8c222df435d94b36cbd13d0830 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -1,13 +1,16 @@ [package] name = "sc-client-db" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] parking_lot = "0.10.0" log = "0.4.8" +rand = "0.7" kvdb = "0.4.0" kvdb-rocksdb = { version = "0.5", optional = true } kvdb-memorydb = "0.4.0" @@ -16,20 +19,20 @@ hash-db = "0.15.2" parity-util-mem = { version = "0.5.1", default-features = false, features = ["std"] } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-client = { version = "0.8", path = "../" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8", path = "../executor" } -sc-state-db = { version = "0.8", path = "../state-db" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-alpha.1", path = "../executor" } +sc-state-db = { version = "0.8.0-alpha.1", path = "../state-db" } +sp-trie = { version = "2.0.0-alpha.1", path = "../../primitives/trie" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } env_logger = "0.7.0" quickcheck = "0.9" kvdb-rocksdb = "0.5" diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs new file mode 100644 index 0000000000000000000000000000000000000000..9858a5c148bfaa13f34af57ce4091540e1b39928 --- /dev/null +++ b/client/db/src/bench.rs @@ -0,0 +1,286 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! State backend that's useful for benchmarking + +use std::sync::Arc; +use std::path::PathBuf; +use std::cell::{Cell, RefCell}; +use rand::Rng; + +use hash_db::{Prefix, Hasher}; +use sp_trie::{MemoryDB, prefixed_key}; +use sp_core::storage::ChildInfo; +use sp_runtime::traits::{Block as BlockT, HasherFor}; +use sp_runtime::Storage; +use sp_state_machine::{DBValue, backend::Backend as StateBackend}; +use kvdb::{KeyValueDB, DBTransaction}; +use kvdb_rocksdb::{Database, DatabaseConfig}; + +type DbState = sp_state_machine::TrieBackend< + Arc>>, HasherFor +>; + +struct StorageDb { + db: Arc, + _block: std::marker::PhantomData, +} + +impl sp_state_machine::Storage> for StorageDb { + fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { + let key = prefixed_key::>(key, prefix); + self.db.get(0, &key) + .map_err(|e| format!("Database backend error: {:?}", e)) + } +} + +/// State that manages the backend database reference. Allows runtime to control the database. +pub struct BenchmarkingState { + path: PathBuf, + root: Cell, + state: RefCell>>, + db: Cell>>, + genesis: as StateBackend>>::Transaction, +} + +impl BenchmarkingState { + /// Create a new instance that creates a database in a temporary dir. + pub fn new(genesis: Storage) -> Result { + let temp_dir = PathBuf::from(std::env::temp_dir()); + let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect(); + let path = temp_dir.join(&name); + + let mut root = B::Hash::default(); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + + std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?; + let mut state = BenchmarkingState { + state: RefCell::new(None), + db: Cell::new(None), + path, + root: Cell::new(root), + genesis: Default::default(), + }; + + state.reopen()?; + let child_delta = genesis.children.into_iter().map(|(storage_key, child_content)| ( + storage_key, + child_content.data.into_iter().map(|(k, v)| (k, Some(v))), + child_content.child_info + )); + let (root, transaction) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( + genesis.top.into_iter().map(|(k, v)| (k, Some(v))), + child_delta, + ); + state.genesis = transaction.clone(); + state.commit(root, transaction)?; + Ok(state) + } + + fn reopen(&self) -> Result<(), String> { + *self.state.borrow_mut() = None; + self.db.set(None); + let db_config = DatabaseConfig::with_columns(1); + let path = self.path.to_str() + .ok_or_else(|| String::from("Invalid database path"))?; + let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?); + self.db.set(Some(db.clone())); + let storage_db = Arc::new(StorageDb:: { db, _block: Default::default() }); + *self.state.borrow_mut() = Some(DbState::::new(storage_db, self.root.get())); + Ok(()) + } + + fn kill(&self) -> Result<(), String> { + self.db.set(None); + *self.state.borrow_mut() = None; + let mut root = B::Hash::default(); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + self.root.set(root); + + std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into()) + } +} + +impl Drop for BenchmarkingState { + fn drop(&mut self) { + self.kill().ok(); + } +} + +fn state_err() -> String { + "State is not open".into() +} + +impl StateBackend> for BenchmarkingState { + type Error = as StateBackend>>::Error; + type Transaction = as StateBackend>>::Transaction; + type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key) + } + + fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key) + } + + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(storage_key, child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> Result { + self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) + } + + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(storage_key, child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key) + } + + fn next_child_storage_key( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(storage_key, child_info, key) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + if let Some(ref state) = *self.state.borrow() { + state.for_keys_with_prefix(prefix, f) + } + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { + if let Some(ref state) = *self.state.borrow() { + state.for_key_values_with_prefix(prefix, f) + } + } + + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + if let Some(ref state) = *self.state.borrow() { + state.for_keys_in_child_storage(storage_key, child_info, f) + } + } + + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + if let Some(ref state) = *self.state.borrow() { + state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + } + } + + fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) where + I: IntoIterator, Option>)> + { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta)) + } + + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (B::Hash, bool, Self::Transaction) where + I: IntoIterator, Option>)>, + { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(storage_key, child_info, delta)) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.pairs()) + } + + fn keys(&self, prefix: &[u8]) -> Vec> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.keys(prefix)) + } + + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(storage_key, child_info, prefix)) + } + + fn as_trie_backend(&mut self) + -> Option<&sp_state_machine::TrieBackend>> + { + None + } + + fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) + -> Result<(), Self::Error> + { + if let Some(db) = self.db.take() { + let mut db_transaction = DBTransaction::new(); + + for (key, (val, rc)) in transaction.drain() { + if rc > 0 { + db_transaction.put(0, &key, &val); + } else if rc < 0 { + db_transaction.delete(0, &key); + } + } + db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; + self.root.set(storage_root); + } else { + return Err("Trying to commit to a closed db".into()) + } + self.reopen() + } + + fn wipe(&self) -> Result<(), Self::Error> { + self.kill()?; + self.reopen()?; + self.commit(self.root.get(), self.genesis.clone())?; + Ok(()) + } +} + +impl std::fmt::Debug for BenchmarkingState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DB at {:?}", self.path) + } +} + diff --git a/client/db/src/cache/list_cache.rs b/client/db/src/cache/list_cache.rs index 72278a1e85e6dd7a293f569084d1337689aba9a1..f3a8171342c9158997cb1dd08e3d90aa03e94b8a 100644 --- a/client/db/src/cache/list_cache.rs +++ b/client/db/src/cache/list_cache.rs @@ -357,7 +357,7 @@ impl> ListCache // it is possible that we're inserting extra (but still required) fork here let new_storage_entry = StorageEntry { prev_valid_from: Some(prev_valid_from), - value: value.expect("chcecked abpve that !value.is_none(); qed"), + value: value.expect("checked above that !value.is_none(); qed"), }; tx.insert_storage_entry(&block, &new_storage_entry); diff --git a/client/db/src/cache/list_entry.rs b/client/db/src/cache/list_entry.rs index d3f7dd57693c1906974c66183f1b6faea36ff523..e18434329079b68d1bc4a13af4a06698dd88d9d8 100644 --- a/client/db/src/cache/list_entry.rs +++ b/client/db/src/cache/list_entry.rs @@ -69,7 +69,7 @@ impl Entry { .map(|(entry, next)| (entry.valid_from, next))) } - /// Searches the list, ending with THIS entry for the best entry preceeding (or at) + /// Searches the list, ending with THIS entry for the best entry preceding (or at) /// given block number. /// If the entry is found, result is the entry and the block id of next entry (if exists). /// NOTE that this function does not check that the passed block is actually linked to diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 72163a56942132ac73f203d204b6721be46f7f98..99488bbaed0964d4509e38b94c37d68dbe2cfb24 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -48,7 +48,7 @@ pub fn extract_new_configuration(header: &Header) -> Option<&Op /// Opaque configuration cache transaction. During its lifetime, no-one should modify cache. This is currently /// guaranteed because import lock is held during block import/finalization. pub struct DbChangesTrieStorageTransaction { - /// Cache operations that must be performed after db transaction is comitted. + /// Cache operations that must be performed after db transaction is committed. cache_ops: DbCacheTransactionOps, /// New configuration (if changed at current block). new_config: Option>, diff --git a/client/db/src/children.rs b/client/db/src/children.rs index c90af66027fdce5ef4e202ab7f9c94dec3f28f8d..2ef67de6a83e1116d20483982ff284ecbd5adf08 100644 --- a/client/db/src/children.rs +++ b/client/db/src/children.rs @@ -100,7 +100,7 @@ mod tests { children2.push(1_6); write_children(&mut tx, 0, PREFIX, 1_2, children2); - db.write(tx.clone()).expect("(2) Commiting transaction failed"); + db.write(tx.clone()).expect("(2) Committing transaction failed"); let r1: Vec = read_children(&db, 0, PREFIX, 1_1).expect("(1) Getting r1 failed"); let r2: Vec = read_children(&db, 0, PREFIX, 1_2).expect("(1) Getting r2 failed"); @@ -109,7 +109,7 @@ mod tests { assert_eq!(r2, vec![1_4, 1_6]); remove_children(&mut tx, 0, PREFIX, 1_2); - db.write(tx).expect("(2) Commiting transaction failed"); + db.write(tx).expect("(2) Committing transaction failed"); let r1: Vec = read_children(&db, 0, PREFIX, 1_1).expect("(2) Getting r1 failed"); let r2: Vec = read_children(&db, 0, PREFIX, 1_2).expect("(2) Getting r2 failed"); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index be569194972cc80a1f26b9505a9d256f02381769..5173497509cf9ad909813d0baa17df72fe406979 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -29,6 +29,9 @@ pub mod light; pub mod offchain; +#[cfg(any(feature = "kvdb-rocksdb", test))] +pub mod bench; + mod children; mod cache; mod changes_tries_storage; @@ -80,6 +83,9 @@ use crate::stats::StateUsageStats; use log::{trace, debug, warn}; pub use sc_state_db::PruningMode; +#[cfg(any(feature = "kvdb-rocksdb", test))] +pub use bench::BenchmarkingState; + #[cfg(feature = "test-helpers")] use sc_client::in_mem::Backend as InMemoryBackend; @@ -699,7 +705,7 @@ impl sp_state_machine::Storage> for DbGenesisSto /// Used as inner structure under lock in `FrozenForDuration`. struct Frozen { at: std::time::Instant, - value: T, + value: Option, } /// Some value frozen for period of time. @@ -709,33 +715,33 @@ struct Frozen { /// a new value which will be again frozen for `duration`. pub(crate) struct FrozenForDuration { duration: std::time::Duration, - value: RwLock>, + value: parking_lot::Mutex>, } impl FrozenForDuration { - fn new(duration: std::time::Duration, initial: T) -> Self { + fn new(duration: std::time::Duration) -> Self { Self { duration, - value: Frozen { at: std::time::Instant::now(), value: initial }.into(), + value: Frozen { at: std::time::Instant::now(), value: None }.into(), } } fn take_or_else(&self, f: F) -> T where F: FnOnce() -> T { - if self.value.read().at.elapsed() > self.duration { - let mut write_lock = self.value.write(); + let mut lock = self.value.lock(); + if lock.at.elapsed() > self.duration || lock.value.is_none() { let new_value = f(); - write_lock.at = std::time::Instant::now(); - write_lock.value = new_value.clone(); + lock.at = std::time::Instant::now(); + lock.value = Some(new_value.clone()); new_value } else { - self.value.read().value.clone() + lock.value.as_ref().expect("checked with lock above").clone() } } } /// Disk backend. /// -/// Disk backend keps data in a key-value store. In archive mode, trie nodes are kept from all blocks. +/// Disk backend keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks. /// Otherwise, trie nodes are kept only from some recent blocks. pub struct Backend { storage: Arc>, @@ -818,7 +824,7 @@ impl Backend { ), import_lock: Default::default(), is_archive: is_archive_pruning, - io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), (kvdb::IoStats::empty(), StateUsageInfo::empty())), + io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), state_usage: StateUsageStats::new(), }) } @@ -872,7 +878,7 @@ impl Backend { inmem } - /// Returns total numbet of blocks (headers) in the block DB. + /// Returns total number of blocks (headers) in the block DB. #[cfg(feature = "test-helpers")] pub fn blocks_count(&self) -> u64 { self.blockchain.db.iter(columns::HEADER).count() as u64 @@ -994,7 +1000,7 @@ impl Backend { Ok((*hash, number, false, true)) } - // performs forced canonicaliziation with a delay after importing a non-finalized block. + // performs forced canonicalization with a delay after importing a non-finalized block. fn force_delayed_canonicalize( &self, transaction: &mut DBTransaction, @@ -1466,6 +1472,7 @@ impl sc_client_api::backend::Backend for Backend { average_transaction_size: io_stats.avg_transaction_size() as u64, state_reads: state_stats.reads.ops, state_reads_cache: state_stats.cache_reads.ops, + state_writes: state_stats.writes.ops, }, }) } diff --git a/client/db/src/light.rs b/client/db/src/light.rs index e663fc569949091fbc8be0bb5163294d610909aa..14ce6ac0f9a05d5511698b169a117d8fbd888d9a 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -102,7 +102,7 @@ impl LightStorage { cache: Arc::new(DbCacheSync(RwLock::new(cache))), header_metadata_cache: HeaderMetadataCache::default(), #[cfg(not(target_os = "unknown"))] - io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1), kvdb::IoStats::empty()), + io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), }) } @@ -592,6 +592,7 @@ impl LightBlockchainStorage for LightStorage // Light client does not track those state_reads: 0, state_reads_cache: 0, + state_writes: 0, } }) } @@ -814,7 +815,7 @@ pub(crate) mod tests { } #[test] - fn get_cht_fails_for_non_existant_cht() { + fn get_cht_fails_for_non_existent_cht() { let cht_size: u64 = cht::size(); assert!(LightStorage::::new_test().header_cht_root(cht_size, cht_size / 2).unwrap().is_none()); } diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index fd85a899b628ef78b377e3035e4f7dc370319a12..6ef29f47b8c42c1b5c5702cc9724da611d87d2f2 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -338,7 +338,7 @@ impl CacheChanges { is_best, ); let cache = &mut *cache; - // Filter out commiting block if any. + // Filter out committing block if any. let enacted: Vec<_> = enacted .iter() .filter(|h| commit_hash.as_ref().map_or(true, |p| *h != p)) @@ -1453,7 +1453,7 @@ mod qc { self.head_state( self.canon.last() - .expect("wasn't forking to emptiness so there shoud be one!") + .expect("wasn't forking to emptiness so there should be one!") .hash ) }, diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 69230e78519258f0fa55424b72a31b6c045627ce..971acf8456b6463fe315768f16e2f8cf3af8254b 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -118,7 +118,7 @@ fn current_version(path: &Path) -> sp_blockchain::Result { } } -/// Opens database of givent type with given number of columns. +/// Opens database of given type with given number of columns. fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result { let db_path = db_path.to_str() .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index f7f51d7f6de5d8dda4e2306c9adae590feede603..f26714eb5a7dbf9edae0fcac6a850e877478fd0e 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -36,6 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc}; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. +#[cfg(any(feature = "kvdb-rocksdb", feature = "test-helpers", test))] pub const NUM_COLUMNS: u32 = 11; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: u32 = 0; @@ -320,7 +321,9 @@ pub fn require_header( id: BlockId, ) -> sp_blockchain::Result { read_header(db, col_index, col, id) - .and_then(|header| header.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("{}", id)))) + .and_then(|header| header.ok_or_else(|| + sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id)) + )) } /// Read meta from the database. diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index c8774470febf3ea3ed82e380b8f0775dd7cbd55c..559a2d8b929d1b705d44aab3e4c944c3bd5a6edb 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -1,28 +1,30 @@ [package] name = "sc-executor" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-serializer = { version = "2.0.0", path = "../../primitives/serializer" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } -sp-panic-handler = { version = "2.0.0", path = "../../primitives/panic-handler" } +sp-io = { version = "2.0.0-alpha.1", path = "../../primitives/io" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-trie = { version = "2.0.0-alpha.1", path = "../../primitives/trie" } +sp-serializer = { version = "2.0.0-alpha.1", path = "../../primitives/serializer" } +sp-version = { version = "2.0.0-alpha.1", path = "../../primitives/version" } +sp-panic-handler = { version = "2.0.0-alpha.1", path = "../../primitives/panic-handler" } wasmi = "0.6.2" parity-wasm = "0.41.0" lazy_static = "1.4.0" -sp-wasm-interface = { version = "2.0.0", path = "../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../primitives/runtime-interface" } -sp-externalities = { version = "0.8.0", path = "../../primitives/externalities" } -sc-executor-common = { version = "0.8", path = "common" } -sc-executor-wasmi = { version = "0.8", path = "wasmi" } -sc-executor-wasmtime = { version = "0.8", path = "wasmtime", optional = true } +sp-wasm-interface = { version = "2.0.0-alpha.1", path = "../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.1", path = "../../primitives/runtime-interface" } +sp-externalities = { version = "0.8.0-alpha.1", path = "../../primitives/externalities" } +sc-executor-common = { version = "0.8.0-alpha.1", path = "common" } +sc-executor-wasmi = { version = "0.8.0-alpha.1", path = "wasmi" } +sc-executor-wasmtime = { version = "0.8.0-alpha.1", path = "wasmtime", optional = true } parking_lot = "0.10.0" log = "0.4.8" libsecp256k1 = "0.3.4" @@ -31,9 +33,9 @@ libsecp256k1 = "0.3.4" assert_matches = "1.3.0" wabt = "0.9.2" hex-literal = "0.2.1" -sc-runtime-test = { version = "2.0.0", path = "runtime-test" } -substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } +sc-runtime-test = { version = "2.0.0-dev", path = "runtime-test" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } test-case = "0.3.3" [features] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index e10afe3448b7e960571c7cb1135f97d72a7cf066..474b05bcccf94fdf201e33f3e9dbd363f1e9c9f4 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -1,20 +1,22 @@ [package] name = "sc-executor-common" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] log = "0.4.8" derive_more = "0.99.2" codec = { package = "parity-scale-codec", version = "1.0.0" } wasmi = "0.6.2" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-serializer = { version = "2.0.0", path = "../../../primitives/serializer" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.1", path = "../../../primitives/allocator" } +sp-wasm-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime-interface" } +sp-serializer = { version = "2.0.0-alpha.1", path = "../../../primitives/serializer" } [features] default = [] diff --git a/client/executor/common/src/error.rs b/client/executor/common/src/error.rs index b8d40d4d91f15c8556953555e4b0f365d89c949b..66d520e942411d01c540bd64f7bde4147be28c74 100644 --- a/client/executor/common/src/error.rs +++ b/client/executor/common/src/error.rs @@ -27,7 +27,7 @@ pub type Result = std::result::Result; pub enum Error { /// Unserializable Data InvalidData(sp_serializer::Error), - /// Trap occured during execution + /// Trap occurred during execution Trap(wasmi::Trap), /// Wasmi loading/instantiating error Wasmi(wasmi::Error), diff --git a/client/executor/common/src/sandbox.rs b/client/executor/common/src/sandbox.rs index f920a47ca7655c2b3a71a8575f65136d9af272fa..ccfdc2f3e0e28c2e8b6a1b1734ee8be450b088ff 100644 --- a/client/executor/common/src/sandbox.rs +++ b/client/executor/common/src/sandbox.rs @@ -498,7 +498,7 @@ pub fn instantiate<'a, FE: SandboxCapabilities>( /// /// This is generic over a supervisor function reference type. pub struct Store { - // Memories and instances are `Some` untill torndown. + // Memories and instances are `Some` until torn down. instances: Vec>>>, memories: Vec>, } diff --git a/client/executor/common/src/wasm_runtime.rs b/client/executor/common/src/wasm_runtime.rs index 0733350f4cd6a45a1a788502bf5a21b904464ed3..7af6c2bd53c0ec1f9de45356d459a100e2cf50e8 100644 --- a/client/executor/common/src/wasm_runtime.rs +++ b/client/executor/common/src/wasm_runtime.rs @@ -17,21 +17,18 @@ //! Definitions for a wasm runtime. use crate::error::Error; -use sp_wasm_interface::Function; +use sp_wasm_interface::{Function, Value}; /// A trait that defines an abstract wasm runtime. /// /// This can be implemented by an execution engine. pub trait WasmRuntime { - /// Attempt to update the number of heap pages available during execution. - /// - /// Returns false if the update cannot be applied. The function is guaranteed to return true if - /// the heap pages would not change from its current value. - fn update_heap_pages(&mut self, heap_pages: u64) -> bool; - /// Return the host functions that are registered for this Wasm runtime. fn host_functions(&self) -> &[&'static dyn Function]; /// Call a method in the Substrate runtime by name. Returns the encoded result on success. fn call(&mut self, method: &str, data: &[u8]) -> Result, Error>; + + /// Get the value from a global with the given `name`. + fn get_global_val(&self, name: &str) -> Result, Error>; } diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 3cb0c03a87c8eea3832ef3993fc6711d58deee86..6964f7d11265be536d06d9602b32e82ca83f7a44 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "sc-runtime-test" -version = "2.0.0" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" } -sp-sandbox = { version = "0.8.0", default-features = false, path = "../../../primitives/sandbox" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-allocator = { version = "2.0.0", default-features = false, path = "../../../primitives/allocator" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/io" } +sp-sandbox = { version = "0.8.0-alpha.1", default-features = false, path = "../../../primitives/sandbox" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/runtime" } +sp-allocator = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/allocator" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [features] default = [ "std" ] diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index b183398c02f09d8751289cbd1a169677a82b6a6f..38a16ae39ea012806d12d90f1a920e622eec4ad0 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -275,6 +275,20 @@ sp_core::wasm_export_functions! { data.to_vec() } + + // Check that the heap at `heap_base + offset` don't contains the test message. + // After the check succeeds the test message is written into the heap. + // + // It is expected that the given pointer is not allocated. + fn check_and_set_in_heap(heap_base: u32, offset: u32) { + let test_message = b"Hello invalid heap memory"; + let ptr = unsafe { (heap_base + offset) as *mut u8 }; + + let message_slice = unsafe { sp_std::slice::from_raw_parts_mut(ptr, test_message.len()) }; + + assert_ne!(test_message, message_slice); + message_slice.copy_from_slice(test_message); + } } #[cfg(not(feature = "std"))] diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 3f3d9f69e13dc329983e1da517a8e9e4dc47c465..c0516d3ac7dfaa8b3447faf234834791efb56842 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -88,7 +88,7 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 68\")" + "Other(\"Wasm execution trapped: call to a missing function env:missing_external\")" ), } } @@ -117,7 +117,7 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 69\")" + "Other(\"Wasm execution trapped: call to a missing function env:yet_another_missing_external\")" ), } } @@ -565,3 +565,26 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) { let res = instance.call("allocates_huge_stack_array", &false.encode()); assert!(res.is_ok()); } + +#[test_case(WasmExecutionMethod::Interpreted)] +fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) { + let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + wasm_method, + 1024, + &WASM_BINARY[..], + HostFunctions::host_functions(), + true, + ).expect("Creates instance"); + + let heap_base = instance.get_global_val("__heap_base") + .expect("`__heap_base` is valid") + .expect("`__heap_base` exists") + .as_i32() + .expect("`__heap_base` is an `i32`"); + + let params = (heap_base as u32, 512u32 * 64 * 1024).encode(); + instance.call("check_and_set_in_heap", ¶ms).unwrap(); + + // Cal it a second time to check that the heap was freed. + instance.call("check_and_set_in_heap", ¶ms).unwrap(); +} diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index b8966c3af27393a5f82076901be24121ab781ce7..9d54246ee07638562fd7ae95ab2cbd0e9efb808a 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -42,6 +42,8 @@ pub enum WasmExecutionMethod { /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { runtime: Box, + /// The number of WebAssembly heap pages this instance was created with. + heap_pages: u64, /// Runtime version according to `Core_version`. version: RuntimeVersion, } @@ -122,7 +124,7 @@ impl RuntimesCache { Entry::Occupied(o) => { let result = o.into_mut(); if let Ok(ref mut cached_runtime) = result { - let heap_pages_changed = !cached_runtime.runtime.update_heap_pages(heap_pages); + let heap_pages_changed = cached_runtime.heap_pages != heap_pages; let host_functions_changed = cached_runtime.runtime.host_functions() != host_functions; if heap_pages_changed || host_functions_changed { @@ -236,6 +238,7 @@ fn create_versioned_wasm_runtime( Ok(VersionedRuntime { runtime, version, + heap_pages, }) } diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index 9e968fdc685f0896abb3b7593fc425e2e5af6c17..da9c4cb598ae55e351126157ecc239d0624a10ef 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -1,17 +1,19 @@ [package] name = "sc-executor-wasmi" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] log = "0.4.8" wasmi = "0.6.2" parity-wasm = "0.41.0" codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor-common = { version = "0.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } +sc-executor-common = { version = "0.8.0-alpha.1", path = "../common" } +sp-wasm-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.1", path = "../../../primitives/allocator" } diff --git a/client/executor/wasmi/src/lib.rs b/client/executor/wasmi/src/lib.rs index 6fbfdbc1cced4f51faf7bf71640305f95cf87103..a0e11dfcf8b93f4799624688e84c8e109f98ca9d 100644 --- a/client/executor/wasmi/src/lib.rs +++ b/client/executor/wasmi/src/lib.rs @@ -73,8 +73,7 @@ impl<'a> sandbox::SandboxCapabilities for FunctionExecutor<'a> { invoke_args_len: WordSize, state: u32, func_idx: sandbox::SupervisorFuncIndex, - ) -> Result - { + ) -> Result { let result = wasmi::FuncInstance::invoke( dispatch_thunk, &[ @@ -277,7 +276,7 @@ struct Resolver<'a> { /// Will be used as initial and maximum size of the imported memory. heap_pages: usize, /// By default, runtimes should import memory and this is `Some(_)` after - /// reolving. However, to be backwards compatible, we also support memory + /// resolving. However, to be backwards compatible, we also support memory /// exported by the WASM blob (this will be `None` after resolving). import_memory: RefCell>, } @@ -536,7 +535,6 @@ struct StateSnapshot { data_segments: Vec<(u32, Vec)>, /// The list of all global mutable variables of the module in their sequential order. global_mut_values: Vec, - heap_pages: u64, } impl StateSnapshot { @@ -544,7 +542,6 @@ impl StateSnapshot { fn take( module_instance: &ModuleRef, data_segments: Vec, - heap_pages: u64, ) -> Option { let prepared_segments = data_segments .into_iter() @@ -590,7 +587,6 @@ impl StateSnapshot { Some(Self { data_segments: prepared_segments, global_mut_values, - heap_pages, }) } @@ -646,10 +642,6 @@ pub struct WasmiRuntime { } impl WasmRuntime for WasmiRuntime { - fn update_heap_pages(&mut self, heap_pages: u64) -> bool { - self.state_snapshot.heap_pages == heap_pages - } - fn host_functions(&self) -> &[&'static dyn Function] { &self.host_functions } @@ -677,6 +669,19 @@ impl WasmRuntime for WasmiRuntime { &self.missing_functions, ) } + + fn get_global_val(&self, name: &str) -> Result, Error> { + match self.instance.export_by_name(name) { + Some(global) => Ok(Some( + global + .as_global() + .ok_or_else(|| format!("`{}` is not a global", name))? + .get() + .into() + )), + None => Ok(None), + } + } } pub fn create_instance( @@ -702,7 +707,7 @@ pub fn create_instance( ).map_err(|e| WasmError::Instantiation(e.to_string()))?; // Take state snapshot before executing anything. - let state_snapshot = StateSnapshot::take(&instance, data_segments, heap_pages) + let state_snapshot = StateSnapshot::take(&instance, data_segments) .expect( "`take` returns `Err` if the module is not valid; we already loaded module above, thus the `Module` is proven to be valid at this point; diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 30d3a5dc87f0e5e6a85d73b908dab47c1d2c721b..709da3525302555d2febdaa7e83d85e6bd327a36 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -1,29 +1,24 @@ [package] name = "sc-executor-wasmtime" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] log = "0.4.8" wasmi = "0.6.2" parity-wasm = "0.41.0" codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor-common = { version = "0.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } +sc-executor-common = { version = "0.8.0-alpha.1", path = "../common" } +sp-wasm-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.1", path = "../../../primitives/allocator" } -cranelift-codegen = "0.50" -cranelift-entity = "0.50" -cranelift-frontend = "0.50" -cranelift-native = "0.50" -cranelift-wasm = "0.50" -wasmtime-environ = "0.8" -wasmtime-jit = "0.8" -wasmtime-runtime = "0.8" +wasmtime = "0.11" [dev-dependencies] assert_matches = "1.3.0" diff --git a/client/executor/wasmtime/src/function_executor.rs b/client/executor/wasmtime/src/function_executor.rs deleted file mode 100644 index b4971f8b8a65e6f32b503efe32d1c55445afdc6a..0000000000000000000000000000000000000000 --- a/client/executor/wasmtime/src/function_executor.rs +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use sp_allocator::FreeingBumpHeapAllocator; -use sc_executor_common::error::{Error, Result}; -use sc_executor_common::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex}; -use crate::util::{ - checked_range, cranelift_ir_signature, read_memory_into, write_memory_from, -}; - -use codec::{Decode, Encode}; -use cranelift_codegen::ir; -use cranelift_codegen::isa::TargetFrontendConfig; -use log::trace; -use sp_core::sandbox as sandbox_primitives; -use std::{cmp, mem, ptr}; -use wasmtime_environ::translate_signature; -use wasmtime_jit::{ActionError, Compiler}; -use wasmtime_runtime::{Export, VMCallerCheckedAnyfunc, VMContext, wasmtime_call_trampoline}; -use sp_wasm_interface::{ - FunctionContext, MemoryId, Pointer, Result as WResult, Sandbox, Signature, Value, ValueType, - WordSize, -}; - -/// Wrapper type for pointer to a Wasm table entry. -/// -/// The wrapper type is used to ensure that the function reference is valid as it must be unsafely -/// dereferenced from within the safe method `::invoke`. -#[derive(Clone, Copy)] -pub struct SupervisorFuncRef(*const VMCallerCheckedAnyfunc); - -/// The state required to construct a FunctionExecutor context. The context only lasts for one host -/// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make -/// many different host calls that must share state. -/// -/// This is stored as part of the host state of the "env" Wasmtime instance. -pub struct FunctionExecutorState { - sandbox_store: sandbox::Store, - heap: FreeingBumpHeapAllocator, -} - -impl FunctionExecutorState { - /// Constructs a new `FunctionExecutorState`. - pub fn new(heap_base: u32) -> Self { - FunctionExecutorState { - sandbox_store: sandbox::Store::new(), - heap: FreeingBumpHeapAllocator::new(heap_base), - } - } - - /// Returns a mutable reference to the heap allocator. - pub fn heap(&mut self) -> &mut FreeingBumpHeapAllocator { - &mut self.heap - } -} - -/// A `FunctionExecutor` implements `FunctionContext` for making host calls from a Wasmtime -/// runtime. The `FunctionExecutor` exists only for the lifetime of the call and borrows state from -/// a longer-living `FunctionExecutorState`. -pub struct FunctionExecutor<'a> { - compiler: &'a mut Compiler, - sandbox_store: &'a mut sandbox::Store, - heap: &'a mut FreeingBumpHeapAllocator, - memory: &'a mut [u8], - table: Option<&'a [VMCallerCheckedAnyfunc]>, -} - -impl<'a> FunctionExecutor<'a> { - /// Construct a new `FunctionExecutor`. - /// - /// The vmctx MUST come from a call to a function in the "env" module. - /// The state MUST be looked up from the host state of the "env" module. - pub unsafe fn new( - vmctx: *mut VMContext, - compiler: &'a mut Compiler, - state: &'a mut FunctionExecutorState, - ) -> Result - { - let memory = match (*vmctx).lookup_global_export("memory") { - Some(Export::Memory { definition, vmctx: _, memory: _ }) => - std::slice::from_raw_parts_mut( - (*definition).base, - (*definition).current_length, - ), - _ => return Err(Error::InvalidMemoryReference), - }; - let table = match (*vmctx).lookup_global_export("__indirect_function_table") { - Some(Export::Table { definition, vmctx: _, table: _ }) => - Some(std::slice::from_raw_parts( - (*definition).base as *const VMCallerCheckedAnyfunc, - (*definition).current_elements as usize, - )), - _ => None, - }; - Ok(FunctionExecutor { - compiler, - sandbox_store: &mut state.sandbox_store, - heap: &mut state.heap, - memory, - table, - }) - } -} - -impl<'a> SandboxCapabilities for FunctionExecutor<'a> { - type SupervisorFuncRef = SupervisorFuncRef; - - fn invoke( - &mut self, - dispatch_thunk: &Self::SupervisorFuncRef, - invoke_args_ptr: Pointer, - invoke_args_len: WordSize, - state: u32, - func_idx: SupervisorFuncIndex, - ) -> Result - { - let func_ptr = unsafe { (*dispatch_thunk.0).func_ptr }; - let vmctx = unsafe { (*dispatch_thunk.0).vmctx }; - - // The following code is based on the wasmtime_jit::Context::invoke. - let value_size = mem::size_of::(); - let (signature, mut values_vec) = generate_signature_and_args( - &[ - Value::I32(u32::from(invoke_args_ptr) as i32), - Value::I32(invoke_args_len as i32), - Value::I32(state as i32), - Value::I32(usize::from(func_idx) as i32), - ], - Some(ValueType::I64), - self.compiler.frontend_config(), - ); - - // Get the trampoline to call for this function. - let exec_code_buf = self.compiler - .get_published_trampoline(func_ptr, &signature, value_size) - .map_err(ActionError::Setup) - .map_err(|e| Error::Other(e.to_string()))?; - - // Call the trampoline. - if let Err(message) = unsafe { - wasmtime_call_trampoline( - vmctx, - exec_code_buf, - values_vec.as_mut_ptr() as *mut u8, - ) - } { - return Err(Error::Other(message)); - } - - // Load the return value out of `values_vec`. - Ok(unsafe { ptr::read(values_vec.as_ptr() as *const i64) }) - } -} - -impl<'a> FunctionContext for FunctionExecutor<'a> { - fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> WResult<()> { - read_memory_into(self.memory, address, dest).map_err(|e| e.to_string()) - } - - fn write_memory(&mut self, address: Pointer, data: &[u8]) -> WResult<()> { - write_memory_from(self.memory, address, data).map_err(|e| e.to_string()) - } - - fn allocate_memory(&mut self, size: WordSize) -> WResult> { - self.heap.allocate(self.memory, size).map_err(|e| e.to_string()) - } - - fn deallocate_memory(&mut self, ptr: Pointer) -> WResult<()> { - self.heap.deallocate(self.memory, ptr).map_err(|e| e.to_string()) - } - - fn sandbox(&mut self) -> &mut dyn Sandbox { - self - } -} - -impl<'a> Sandbox for FunctionExecutor<'a> { - fn memory_get( - &mut self, - memory_id: MemoryId, - offset: WordSize, - buf_ptr: Pointer, - buf_len: WordSize, - ) -> WResult - { - let sandboxed_memory = self.sandbox_store.memory(memory_id) - .map_err(|e| e.to_string())?; - sandboxed_memory.with_direct_access(|memory| { - let len = buf_len as usize; - let src_range = match checked_range(offset as usize, len, memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - let dst_range = match checked_range(buf_ptr.into(), len, self.memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - &mut self.memory[dst_range].copy_from_slice(&memory[src_range]); - Ok(sandbox_primitives::ERR_OK) - }) - } - - fn memory_set( - &mut self, - memory_id: MemoryId, - offset: WordSize, - val_ptr: Pointer, - val_len: WordSize, - ) -> WResult - { - let sandboxed_memory = self.sandbox_store.memory(memory_id) - .map_err(|e| e.to_string())?; - sandboxed_memory.with_direct_access_mut(|memory| { - let len = val_len as usize; - let src_range = match checked_range(val_ptr.into(), len, self.memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - let dst_range = match checked_range(offset as usize, len, memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - &mut memory[dst_range].copy_from_slice(&self.memory[src_range]); - Ok(sandbox_primitives::ERR_OK) - }) - } - - fn memory_teardown(&mut self, memory_id: MemoryId) - -> WResult<()> - { - self.sandbox_store.memory_teardown(memory_id).map_err(|e| e.to_string()) - } - - fn memory_new(&mut self, initial: u32, maximum: MemoryId) -> WResult { - self.sandbox_store.new_memory(initial, maximum).map_err(|e| e.to_string()) - } - - fn invoke( - &mut self, - instance_id: u32, - export_name: &str, - args: &[u8], - return_val: Pointer, - return_val_len: u32, - state: u32, - ) -> WResult { - trace!(target: "sp-sandbox", "invoke, instance_idx={}", instance_id); - - // Deserialize arguments and convert them into wasmi types. - let args = Vec::::decode(&mut &args[..]) - .map_err(|_| "Can't decode serialized arguments for the invocation")? - .into_iter() - .map(Into::into) - .collect::>(); - - let instance = self.sandbox_store.instance(instance_id).map_err(|e| e.to_string())?; - let result = instance.invoke(export_name, &args, self, state); - - match result { - Ok(None) => Ok(sandbox_primitives::ERR_OK), - Ok(Some(val)) => { - // Serialize return value and write it back into the memory. - sp_wasm_interface::ReturnValue::Value(val.into()).using_encoded(|val| { - if val.len() > return_val_len as usize { - Err("Return value buffer is too small")?; - } - FunctionContext::write_memory(self, return_val, val)?; - Ok(sandbox_primitives::ERR_OK) - }) - } - Err(_) => Ok(sandbox_primitives::ERR_EXECUTION), - } - } - - fn instance_teardown(&mut self, instance_id: u32) -> WResult<()> { - self.sandbox_store.instance_teardown(instance_id).map_err(|e| e.to_string()) - } - - fn instance_new(&mut self, dispatch_thunk_id: u32, wasm: &[u8], raw_env_def: &[u8], state: u32) - -> WResult - { - // Extract a dispatch thunk from instance's table by the specified index. - let dispatch_thunk = { - let table = self.table.as_ref() - .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")?; - let func_ref = table.get(dispatch_thunk_id as usize) - .ok_or_else(|| "dispatch_thunk_idx is out of the table bounds")?; - SupervisorFuncRef(func_ref) - }; - - let guest_env = match sandbox::GuestEnvironment::decode(&self.sandbox_store, raw_env_def) { - Ok(guest_env) => guest_env, - Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), - }; - - let instance_idx_or_err_code = - match sandbox::instantiate(self, dispatch_thunk, wasm, guest_env, state) - .map(|i| i.register(&mut self.sandbox_store)) - { - Ok(instance_idx) => instance_idx, - Err(sandbox::InstantiationError::StartTrapped) => - sandbox_primitives::ERR_EXECUTION, - Err(_) => sandbox_primitives::ERR_MODULE, - }; - - Ok(instance_idx_or_err_code as u32) - } - - fn get_global_val( - &self, - instance_idx: u32, - name: &str, - ) -> WResult> { - self.sandbox_store - .instance(instance_idx) - .map(|i| i.get_global_val(name)) - .map_err(|e| e.to_string()) - } -} - -// The storage for a Wasmtime invocation argument. -#[derive(Debug, Default, Copy, Clone)] -#[repr(C, align(8))] -struct VMInvokeArgument([u8; 8]); - -fn generate_signature_and_args( - args: &[Value], - result_type: Option, - frontend_config: TargetFrontendConfig, -) -> (ir::Signature, Vec) -{ - // This code is based on the wasmtime_jit::Context::invoke. - - let param_types = args.iter() - .map(|arg| arg.value_type()) - .collect::>(); - let signature = translate_signature( - cranelift_ir_signature( - Signature::new(param_types, result_type), - &frontend_config.default_call_conv - ), - frontend_config.pointer_type() - ); - - let mut values_vec = vec![ - VMInvokeArgument::default(); - cmp::max(args.len(), result_type.iter().len()) - ]; - - // Store the argument values into `values_vec`. - for (index, arg) in args.iter().enumerate() { - unsafe { - let ptr = values_vec.as_mut_ptr().add(index); - - match arg { - Value::I32(x) => ptr::write(ptr as *mut i32, *x), - Value::I64(x) => ptr::write(ptr as *mut i64, *x), - Value::F32(x) => ptr::write(ptr as *mut u32, *x), - Value::F64(x) => ptr::write(ptr as *mut u64, *x), - } - } - } - - (signature, values_vec) -} - diff --git a/client/executor/wasmtime/src/host.rs b/client/executor/wasmtime/src/host.rs new file mode 100644 index 0000000000000000000000000000000000000000..e0cc6ecc9ae7a1dd738352a0d287ff9d5ad7ff5b --- /dev/null +++ b/client/executor/wasmtime/src/host.rs @@ -0,0 +1,349 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! This module defines `HostState` and `HostContext` structs which provide logic and state +//! required for execution of host. + +use crate::instance_wrapper::InstanceWrapper; +use crate::util; +use std::cell::RefCell; +use log::trace; +use codec::{Encode, Decode}; +use sp_allocator::FreeingBumpHeapAllocator; +use sc_executor_common::error::Result; +use sc_executor_common::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex}; +use sp_core::sandbox as sandbox_primitives; +use sp_wasm_interface::{FunctionContext, MemoryId, Pointer, Sandbox, WordSize}; +use wasmtime::{Func, Val}; + +/// Wrapper type for pointer to a Wasm table entry. +/// +/// The wrapper type is used to ensure that the function reference is valid as it must be unsafely +/// dereferenced from within the safe method `::invoke`. +#[derive(Clone)] +pub struct SupervisorFuncRef(Func); + +/// The state required to construct a HostContext context. The context only lasts for one host +/// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make +/// many different host calls that must share state. +pub struct HostState { + // We need some interior mutability here since the host state is shared between all host + // function handlers and the wasmtime backend's `impl WasmRuntime`. + // + // Furthermore, because of recursive calls (e.g. runtime can create and call an sandboxed + // instance which in turn can call the runtime back) we have to be very careful with borrowing + // those. + // + // Basically, most of the interactions should do temporary borrow immediately releasing the + // borrow after performing necessary queries/changes. + sandbox_store: RefCell>, + allocator: RefCell, + instance: InstanceWrapper, +} + +impl HostState { + /// Constructs a new `HostState`. + pub fn new(allocator: FreeingBumpHeapAllocator, instance: InstanceWrapper) -> Self { + HostState { + sandbox_store: RefCell::new(sandbox::Store::new()), + allocator: RefCell::new(allocator), + instance, + } + } + + /// Destruct the host state and extract the `InstanceWrapper` passed at the creation. + pub fn into_instance(self) -> InstanceWrapper { + self.instance + } + + /// Materialize `HostContext` that can be used to invoke a substrate host `dyn Function`. + pub fn materialize<'a>(&'a self) -> HostContext<'a> { + HostContext(self) + } +} + +/// A `HostContext` implements `FunctionContext` for making host calls from a Wasmtime +/// runtime. The `HostContext` exists only for the lifetime of the call and borrows state from +/// a longer-living `HostState`. +pub struct HostContext<'a>(&'a HostState); + +impl<'a> std::ops::Deref for HostContext<'a> { + type Target = HostState; + fn deref(&self) -> &HostState { + self.0 + } +} + +impl<'a> SandboxCapabilities for HostContext<'a> { + type SupervisorFuncRef = SupervisorFuncRef; + + fn invoke( + &mut self, + dispatch_thunk: &Self::SupervisorFuncRef, + invoke_args_ptr: Pointer, + invoke_args_len: WordSize, + state: u32, + func_idx: SupervisorFuncIndex, + ) -> Result { + let result = dispatch_thunk.0.call(&[ + Val::I32(u32::from(invoke_args_ptr) as i32), + Val::I32(invoke_args_len as i32), + Val::I32(state as i32), + Val::I32(usize::from(func_idx) as i32), + ]); + match result { + Ok(ret_vals) => { + let ret_val = if ret_vals.len() != 1 { + return Err(format!( + "Supervisor function returned {} results, expected 1", + ret_vals.len() + ) + .into()); + } else { + &ret_vals[0] + }; + + if let Some(ret_val) = ret_val.i64() { + Ok(ret_val) + } else { + return Err("Supervisor function returned unexpected result!".into()); + } + } + Err(err) => Err(err.message().to_string().into()), + } + } +} + +impl<'a> sp_wasm_interface::FunctionContext for HostContext<'a> { + fn read_memory_into( + &self, + address: Pointer, + dest: &mut [u8], + ) -> sp_wasm_interface::Result<()> { + self.instance + .read_memory_into(address, dest) + .map_err(|e| e.to_string()) + } + + fn write_memory(&mut self, address: Pointer, data: &[u8]) -> sp_wasm_interface::Result<()> { + self.instance + .write_memory_from(address, data) + .map_err(|e| e.to_string()) + } + + fn allocate_memory(&mut self, size: WordSize) -> sp_wasm_interface::Result> { + self.instance + .allocate(&mut *self.allocator.borrow_mut(), size) + .map_err(|e| e.to_string()) + } + + fn deallocate_memory(&mut self, ptr: Pointer) -> sp_wasm_interface::Result<()> { + self.instance + .deallocate(&mut *self.allocator.borrow_mut(), ptr) + .map_err(|e| e.to_string()) + } + + fn sandbox(&mut self) -> &mut dyn Sandbox { + self + } +} + +impl<'a> Sandbox for HostContext<'a> { + fn memory_get( + &mut self, + memory_id: MemoryId, + offset: WordSize, + buf_ptr: Pointer, + buf_len: WordSize, + ) -> sp_wasm_interface::Result { + let sandboxed_memory = self + .sandbox_store + .borrow() + .memory(memory_id) + .map_err(|e| e.to_string())?; + sandboxed_memory.with_direct_access(|sandboxed_memory| { + let len = buf_len as usize; + let src_range = match util::checked_range(offset as usize, len, sandboxed_memory.len()) + { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + let supervisor_mem_size = self.instance.memory_size() as usize; + let dst_range = match util::checked_range(buf_ptr.into(), len, supervisor_mem_size) { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + self.instance + .write_memory_from( + Pointer::new(dst_range.start as u32), + &sandboxed_memory[src_range], + ) + .expect("ranges are checked above; write can't fail; qed"); + Ok(sandbox_primitives::ERR_OK) + }) + } + + fn memory_set( + &mut self, + memory_id: MemoryId, + offset: WordSize, + val_ptr: Pointer, + val_len: WordSize, + ) -> sp_wasm_interface::Result { + let sandboxed_memory = self + .sandbox_store + .borrow() + .memory(memory_id) + .map_err(|e| e.to_string())?; + sandboxed_memory.with_direct_access_mut(|sandboxed_memory| { + let len = val_len as usize; + let supervisor_mem_size = self.instance.memory_size() as usize; + let src_range = match util::checked_range(val_ptr.into(), len, supervisor_mem_size) { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + let dst_range = match util::checked_range(offset as usize, len, sandboxed_memory.len()) + { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + self.instance + .read_memory_into( + Pointer::new(src_range.start as u32), + &mut sandboxed_memory[dst_range], + ) + .expect("ranges are checked above; read can't fail; qed"); + Ok(sandbox_primitives::ERR_OK) + }) + } + + fn memory_teardown(&mut self, memory_id: MemoryId) -> sp_wasm_interface::Result<()> { + self.sandbox_store + .borrow_mut() + .memory_teardown(memory_id) + .map_err(|e| e.to_string()) + } + + fn memory_new(&mut self, initial: u32, maximum: MemoryId) -> sp_wasm_interface::Result { + self.sandbox_store + .borrow_mut() + .new_memory(initial, maximum) + .map_err(|e| e.to_string()) + } + + fn invoke( + &mut self, + instance_id: u32, + export_name: &str, + args: &[u8], + return_val: Pointer, + return_val_len: u32, + state: u32, + ) -> sp_wasm_interface::Result { + trace!(target: "sp-sandbox", "invoke, instance_idx={}", instance_id); + + // Deserialize arguments and convert them into wasmi types. + let args = Vec::::decode(&mut &args[..]) + .map_err(|_| "Can't decode serialized arguments for the invocation")? + .into_iter() + .map(Into::into) + .collect::>(); + + let instance = self + .sandbox_store + .borrow() + .instance(instance_id) + .map_err(|e| e.to_string())?; + let result = instance.invoke(export_name, &args, self, state); + + match result { + Ok(None) => Ok(sandbox_primitives::ERR_OK), + Ok(Some(val)) => { + // Serialize return value and write it back into the memory. + sp_wasm_interface::ReturnValue::Value(val.into()).using_encoded(|val| { + if val.len() > return_val_len as usize { + Err("Return value buffer is too small")?; + } + ::write_memory(self, return_val, val) + .map_err(|_| "can't write return value")?; + Ok(sandbox_primitives::ERR_OK) + }) + } + Err(_) => Ok(sandbox_primitives::ERR_EXECUTION), + } + } + + fn instance_teardown(&mut self, instance_id: u32) -> sp_wasm_interface::Result<()> { + self.sandbox_store + .borrow_mut() + .instance_teardown(instance_id) + .map_err(|e| e.to_string()) + } + + fn instance_new( + &mut self, + dispatch_thunk_id: u32, + wasm: &[u8], + raw_env_def: &[u8], + state: u32, + ) -> sp_wasm_interface::Result { + // Extract a dispatch thunk from the instance's table by the specified index. + let dispatch_thunk = { + let table_item = self + .instance + .table() + .as_ref() + .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")? + .get(dispatch_thunk_id); + + let func_ref = table_item + .ok_or_else(|| "dispatch_thunk_id is out of bounds")? + .funcref() + .ok_or_else(|| "dispatch_thunk_idx should be a funcref")? + .clone(); + SupervisorFuncRef(func_ref) + }; + + let guest_env = + match sandbox::GuestEnvironment::decode(&*self.sandbox_store.borrow(), raw_env_def) { + Ok(guest_env) => guest_env, + Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), + }; + + let instance_idx_or_err_code = + match sandbox::instantiate(self, dispatch_thunk, wasm, guest_env, state) + .map(|i| i.register(&mut *self.sandbox_store.borrow_mut())) + { + Ok(instance_idx) => instance_idx, + Err(sandbox::InstantiationError::StartTrapped) => sandbox_primitives::ERR_EXECUTION, + Err(_) => sandbox_primitives::ERR_MODULE, + }; + + Ok(instance_idx_or_err_code as u32) + } + + fn get_global_val( + &self, + instance_idx: u32, + name: &str, + ) -> sp_wasm_interface::Result> { + self.sandbox_store + .borrow() + .instance(instance_idx) + .map(|i| i.get_global_val(name)) + .map_err(|e| e.to_string()) + } +} diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs new file mode 100644 index 0000000000000000000000000000000000000000..349f84a0d74d27031ea3d3cba613cb3f36f02fbf --- /dev/null +++ b/client/executor/wasmtime/src/imports.rs @@ -0,0 +1,333 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::state_holder::StateHolder; +use sc_executor_common::error::WasmError; +use sp_wasm_interface::{Function, Value, ValueType}; +use std::any::Any; +use std::rc::Rc; +use wasmtime::{ + Callable, Extern, ExternType, Func, FuncType, ImportType, Limits, Memory, MemoryType, Module, + Trap, Val, +}; + +pub struct Imports { + /// Contains the index into `externs` where the memory import is stored if any. `None` if there + /// is none. + pub memory_import_index: Option, + pub externs: Vec, +} + +/// Goes over all imports of a module and prepares a vector of `Extern`s that can be used for +/// instantiation of the module. Returns an error if there are imports that cannot be satisfied. +pub fn resolve_imports( + state_holder: &StateHolder, + module: &Module, + host_functions: &[&'static dyn Function], + heap_pages: u32, + allow_missing_func_imports: bool, +) -> Result { + let mut externs = vec![]; + let mut memory_import_index = None; + for import_ty in module.imports() { + if import_ty.module() != "env" { + return Err(WasmError::Other(format!( + "host doesn't provide any imports from non-env module: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + + let resolved = match import_ty.name() { + "memory" => { + memory_import_index = Some(externs.len()); + resolve_memory_import(module, import_ty, heap_pages)? + } + _ => resolve_func_import( + module, + state_holder, + import_ty, + host_functions, + allow_missing_func_imports, + )?, + }; + externs.push(resolved); + } + Ok(Imports { + memory_import_index, + externs, + }) +} + +fn resolve_memory_import( + module: &Module, + import_ty: &ImportType, + heap_pages: u32, +) -> Result { + let requested_memory_ty = match import_ty.ty() { + ExternType::Memory(memory_ty) => memory_ty, + _ => { + return Err(WasmError::Other(format!( + "this import must be of memory type: {}:{}", + import_ty.module(), + import_ty.name() + ))) + } + }; + + // Increment the min (a.k.a initial) number of pages by `heap_pages` and check if it exceeds the + // maximum specified by the import. + let initial = requested_memory_ty + .limits() + .min() + .saturating_add(heap_pages); + if let Some(max) = requested_memory_ty.limits().max() { + if initial > max { + return Err(WasmError::Other(format!( + "incremented number of pages by heap_pages (total={}) is more than maximum requested\ + by the runtime wasm module {}", + initial, + max, + ))); + } + } + + let memory_ty = MemoryType::new(Limits::new(initial, requested_memory_ty.limits().max())); + let memory = Memory::new(module.store(), memory_ty); + Ok(Extern::Memory(memory)) +} + +fn resolve_func_import( + module: &Module, + state_holder: &StateHolder, + import_ty: &ImportType, + host_functions: &[&'static dyn Function], + allow_missing_func_imports: bool, +) -> Result { + let func_ty = match import_ty.ty() { + ExternType::Func(func_ty) => func_ty, + _ => { + return Err(WasmError::Other(format!( + "host doesn't provide any non function imports besides 'memory': {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + }; + + let host_func = match host_functions + .iter() + .find(|host_func| host_func.name() == import_ty.name()) + { + Some(host_func) => host_func, + None if allow_missing_func_imports => { + return Ok(MissingHostFuncHandler::new(import_ty).into_extern(module, func_ty)); + } + None => { + return Err(WasmError::Other(format!( + "host doesn't provide such function: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + }; + if !signature_matches(&func_ty, &wasmtime_func_sig(*host_func)) { + return Err(WasmError::Other(format!( + "signature mismatch for: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + + Ok(HostFuncHandler::new(&state_holder, *host_func).into_extern(module)) +} + +/// Returns `true` if `lhs` and `rhs` represent the same signature. +fn signature_matches(lhs: &wasmtime::FuncType, rhs: &wasmtime::FuncType) -> bool { + lhs.params() == rhs.params() && lhs.results() == rhs.results() +} + +/// This structure implements `Callable` and acts as a bridge between wasmtime and +/// substrate host functions. +struct HostFuncHandler { + state_holder: StateHolder, + host_func: &'static dyn Function, +} + +impl HostFuncHandler { + fn new(state_holder: &StateHolder, host_func: &'static dyn Function) -> Self { + Self { + state_holder: state_holder.clone(), + host_func, + } + } + + fn into_extern(self, module: &Module) -> Extern { + let func_ty = wasmtime_func_sig(self.host_func); + let func = Func::new(module.store(), func_ty, Rc::new(self)); + Extern::Func(func) + } +} + +impl Callable for HostFuncHandler { + fn call( + &self, + wasmtime_params: &[Val], + wasmtime_results: &mut [Val], + ) -> Result<(), wasmtime::Trap> { + let unwind_result = self.state_holder.with_context(|host_ctx| { + let mut host_ctx = host_ctx.expect( + "host functions can be called only from wasm instance; + wasm instance is always called initializing context; + therefore host_ctx cannot be None; + qed + ", + ); + // `into_value` panics if it encounters a value that doesn't fit into the values + // available in substrate. + // + // This, however, cannot happen since the signature of this function is created from + // a `dyn Function` signature of which cannot have a non substrate value by definition. + let mut params = wasmtime_params.iter().cloned().map(into_value); + + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.host_func.execute(&mut host_ctx, &mut params) + })) + }); + + let execution_result = match unwind_result { + Ok(execution_result) => execution_result, + Err(err) => return Err(Trap::new(stringify_panic_payload(err))), + }; + + match execution_result { + Ok(Some(ret_val)) => { + debug_assert!( + wasmtime_results.len() == 1, + "wasmtime function signature, therefore the number of results, should always \ + correspond to the number of results returned by the host function", + ); + wasmtime_results[0] = into_wasmtime_val(ret_val); + Ok(()) + } + Ok(None) => { + debug_assert!( + wasmtime_results.len() == 0, + "wasmtime function signature, therefore the number of results, should always \ + correspond to the number of results returned by the host function", + ); + Ok(()) + } + Err(msg) => Err(Trap::new(msg)), + } + } +} + +/// A `Callable` handler for missing functions. +struct MissingHostFuncHandler { + module: String, + name: String, +} + +impl MissingHostFuncHandler { + fn new(import_ty: &ImportType) -> Self { + Self { + module: import_ty.module().to_string(), + name: import_ty.name().to_string(), + } + } + + fn into_extern(self, module: &Module, func_ty: &FuncType) -> Extern { + let func = Func::new(module.store(), func_ty.clone(), Rc::new(self)); + Extern::Func(func) + } +} + +impl Callable for MissingHostFuncHandler { + fn call( + &self, + _wasmtime_params: &[Val], + _wasmtime_results: &mut [Val], + ) -> Result<(), wasmtime::Trap> { + Err(Trap::new(format!( + "call to a missing function {}:{}", + self.module, self.name + ))) + } +} + +fn wasmtime_func_sig(func: &dyn Function) -> wasmtime::FuncType { + let params = func + .signature() + .args + .iter() + .cloned() + .map(into_wasmtime_val_type) + .collect::>() + .into_boxed_slice(); + let results = func + .signature() + .return_value + .iter() + .cloned() + .map(into_wasmtime_val_type) + .collect::>() + .into_boxed_slice(); + wasmtime::FuncType::new(params, results) +} + +fn into_wasmtime_val_type(val_ty: ValueType) -> wasmtime::ValType { + match val_ty { + ValueType::I32 => wasmtime::ValType::I32, + ValueType::I64 => wasmtime::ValType::I64, + ValueType::F32 => wasmtime::ValType::F32, + ValueType::F64 => wasmtime::ValType::F64, + } +} + +/// Converts a `Val` into a substrate runtime interface `Value`. +/// +/// Panics if the given value doesn't have a corresponding variant in `Value`. +fn into_value(val: Val) -> Value { + match val { + Val::I32(v) => Value::I32(v), + Val::I64(v) => Value::I64(v), + Val::F32(f_bits) => Value::F32(f_bits), + Val::F64(f_bits) => Value::F64(f_bits), + _ => panic!("Given value type is unsupported by substrate"), + } +} + +fn into_wasmtime_val(value: Value) -> wasmtime::Val { + match value { + Value::I32(v) => Val::I32(v), + Value::I64(v) => Val::I64(v), + Value::F32(f_bits) => Val::F32(f_bits), + Value::F64(f_bits) => Val::F64(f_bits), + } +} + +/// Attempt to convert a opaque panic payload to a string. +fn stringify_panic_payload(payload: Box) -> String { + match payload.downcast::<&'static str>() { + Ok(msg) => msg.to_string(), + Err(payload) => match payload.downcast::() { + Ok(msg) => *msg, + // At least we tried... + Err(_) => "Box".to_string(), + }, + } +} diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs new file mode 100644 index 0000000000000000000000000000000000000000..159746801a52aa68e45c63b4f060a1804415a2f8 --- /dev/null +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -0,0 +1,276 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Defines data and logic needed for interaction with an WebAssembly instance of a substrate +//! runtime module. + +use crate::util; +use crate::imports::Imports; + +use sc_executor_common::error::{Error, Result}; +use sp_wasm_interface::{Pointer, WordSize, Value}; +use std::slice; +use std::marker; +use wasmtime::{Instance, Module, Memory, Table, Val}; + +/// Wrap the given WebAssembly Instance of a wasm module with Substrate-runtime. +/// +/// This struct is a handy wrapper around a wasmtime `Instance` that provides substrate specific +/// routines. +pub struct InstanceWrapper { + instance: Instance, + // The memory instance of the `instance`. + // + // It is important to make sure that we don't make any copies of this to make it easier to proof + // See `memory_as_slice` and `memory_as_slice_mut`. + memory: Memory, + table: Option, + // Make this struct explicitly !Send & !Sync. + _not_send_nor_sync: marker::PhantomData<*const ()>, +} + +impl InstanceWrapper { + /// Create a new instance wrapper from the given wasm module. + pub fn new(module: &Module, imports: &Imports, heap_pages: u32) -> Result { + let instance = Instance::new(module, &imports.externs) + .map_err(|e| Error::from(format!("cannot instantiate: {}", e)))?; + + let memory = match imports.memory_import_index { + Some(memory_idx) => { + imports.externs[memory_idx] + .memory() + .expect("only memory can be at the `memory_idx`; qed") + .clone() + } + None => { + let memory = get_linear_memory(&instance)?; + if !memory.grow(heap_pages).is_ok() { + return Err("failed top increase the linear memory size".into()); + } + memory + }, + }; + + Ok(Self { + table: get_table(&instance), + memory, + instance, + _not_send_nor_sync: marker::PhantomData, + }) + } + + /// Resolves a substrate entrypoint by the given name. + /// + /// An entrypoint must have a signature `(i32, i32) -> i64`, otherwise this function will return + /// an error. + pub fn resolve_entrypoint(&self, name: &str) -> Result { + // Resolve the requested method and verify that it has a proper signature. + let export = self + .instance + .get_export(name) + .ok_or_else(|| Error::from(format!("Exported method {} is not found", name)))?; + let entrypoint = export + .func() + .ok_or_else(|| Error::from(format!("Export {} is not a function", name)))?; + match (entrypoint.ty().params(), entrypoint.ty().results()) { + (&[wasmtime::ValType::I32, wasmtime::ValType::I32], &[wasmtime::ValType::I64]) => {} + _ => { + return Err(Error::from(format!( + "method {} have an unsupported signature", + name + ))) + } + } + Ok(entrypoint.clone()) + } + + /// Returns an indirect function table of this instance. + pub fn table(&self) -> Option<&Table> { + self.table.as_ref() + } + + /// Returns the byte size of the linear memory instance attached to this instance. + pub fn memory_size(&self) -> u32 { + self.memory.data_size() as u32 + } + + /// Reads `__heap_base: i32` global variable and returns it. + /// + /// If it doesn't exist, not a global or of not i32 type returns an error. + pub fn extract_heap_base(&self) -> Result { + let heap_base_export = self + .instance + .get_export("__heap_base") + .ok_or_else(|| Error::from("__heap_base is not found"))?; + + let heap_base_global = heap_base_export + .global() + .ok_or_else(|| Error::from("__heap_base is not a global"))?; + + let heap_base = heap_base_global + .get() + .i32() + .ok_or_else(|| Error::from("__heap_base is not a i32"))?; + + Ok(heap_base as u32) + } + + /// Get the value from a global with the given `name`. + pub fn get_global_val(&self, name: &str) -> Result> { + let global = match self.instance.get_export(name) { + Some(global) => global, + None => return Ok(None), + }; + + let global = global.global().ok_or_else(|| format!("`{}` is not a global", name))?; + + match global.get() { + Val::I32(val) => Ok(Some(Value::I32(val))), + Val::I64(val) => Ok(Some(Value::I64(val))), + Val::F32(val) => Ok(Some(Value::F32(val))), + Val::F64(val) => Ok(Some(Value::F64(val))), + _ => Err("Unknown value type".into()), + } + } +} + +/// Extract linear memory instance from the given instance. +fn get_linear_memory(instance: &Instance) -> Result { + let memory_export = instance + .get_export("memory") + .ok_or_else(|| Error::from("memory is not exported under `memory` name"))?; + + let memory = memory_export + .memory() + .ok_or_else(|| Error::from("the `memory` export should have memory type"))? + .clone(); + + Ok(memory) +} + +/// Extract the table from the given instance if any. +fn get_table(instance: &Instance) -> Option
{ + instance + .get_export("__indirect_function_table") + .and_then(|export| export.table()) + .cloned() +} + +/// Functions realted to memory. +impl InstanceWrapper { + /// Read data from a slice of memory into a destination buffer. + /// + /// Returns an error if the read would go out of the memory bounds. + pub fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice(); + + let range = util::checked_range(address.into(), dest.len(), memory.len()) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + dest.copy_from_slice(&memory[range]); + Ok(()) + } + } + + /// Write data to a slice of memory. + /// + /// Returns an error if the write would go out of the memory bounds. + pub fn write_memory_from(&self, address: Pointer, data: &[u8]) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + let range = util::checked_range(address.into(), data.len(), memory.len()) + .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; + &mut memory[range].copy_from_slice(data); + Ok(()) + } + } + + /// Allocate some memory of the given size. Returns pointer to the allocated memory region. + /// + /// Returns `Err` in case memory cannot be allocated. Refer to the allocator documentation + /// to get more details. + pub fn allocate( + &self, + allocator: &mut sp_allocator::FreeingBumpHeapAllocator, + size: WordSize, + ) -> Result> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + allocator.allocate(memory, size).map_err(Into::into) + } + } + + /// Deallocate the memory pointed by the given pointer. + /// + /// Returns `Err` in case the given memory region cannot be deallocated. + pub fn deallocate( + &self, + allocator: &mut sp_allocator::FreeingBumpHeapAllocator, + ptr: Pointer, + ) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + allocator.deallocate(memory, ptr).map_err(Into::into) + } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// Wasmtime doesn't provide comprehensive documentation about the exact behavior of the data + /// pointer. If a dynamic style heap is used the base pointer of the heap can change. Since + /// growing, we cannot guarantee the lifetime of the returned slice reference. + unsafe fn memory_as_slice(&self) -> &[u8] { + let ptr = self.memory.data_ptr() as *const _; + let len = self.memory.data_size(); + + if len == 0 { + &[] + } else { + slice::from_raw_parts(ptr, len) + } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// See `[memory_as_slice]`. In addition to those requirements, since a mutable reference is + /// returned it must be ensured that only one mutable and no shared references to memory exists + /// at the same time. + unsafe fn memory_as_slice_mut(&self) -> &mut [u8] { + let ptr = self.memory.data_ptr(); + let len = self.memory.data_size(); + + if len == 0 { + &mut [] + } else { + slice::from_raw_parts_mut(ptr, len) + } + } +} diff --git a/client/executor/wasmtime/src/lib.rs b/client/executor/wasmtime/src/lib.rs index 244fca8f842a3083cacbca010f70e53a7a0087eb..8f4801e6da1d0de171ff50d2e8b04c0a08d1f00d 100644 --- a/client/executor/wasmtime/src/lib.rs +++ b/client/executor/wasmtime/src/lib.rs @@ -16,10 +16,11 @@ ///! Defines a `WasmRuntime` that uses the Wasmtime JIT to execute. -mod function_executor; +mod host; mod runtime; -mod trampoline; +mod state_holder; +mod imports; +mod instance_wrapper; mod util; pub use runtime::create_instance; - diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index abf860667f818f3037cb638a6d52fa9fe144aec1..b99d3347872059d778f50c3c00581252e224a167 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -16,70 +16,51 @@ //! Defines the compiled Wasm runtime that uses Wasmtime internally. -use crate::function_executor::FunctionExecutorState; -use crate::trampoline::{EnvState, make_trampoline}; -use crate::util::{ - cranelift_ir_signature, - convert_parity_wasm_signature, - read_memory_into, - write_memory_from -}; +use crate::host::HostState; +use crate::imports::{resolve_imports, Imports}; +use crate::instance_wrapper::InstanceWrapper; +use crate::state_holder::StateHolder; use sc_executor_common::{ error::{Error, Result, WasmError}, wasm_runtime::WasmRuntime, }; -use sp_wasm_interface::{Pointer, WordSize, Function}; +use sp_allocator::FreeingBumpHeapAllocator; use sp_runtime_interface::unpack_ptr_and_len; +use sp_wasm_interface::{Function, Pointer, WordSize, Value}; +use wasmtime::{Config, Engine, Module, Store}; -use std::{cell::RefCell, collections::HashMap, convert::TryFrom, rc::Rc}; - -use cranelift_codegen::ir; -use cranelift_codegen::isa::TargetIsa; -use cranelift_entity::{EntityRef, PrimaryMap}; -use cranelift_frontend::FunctionBuilderContext; -use cranelift_wasm::{DefinedFuncIndex, MemoryIndex}; -use wasmtime_environ::{Module, translate_signature}; -use wasmtime_jit::{ - ActionOutcome, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, RuntimeValue, -}; -use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody}; - -/// TODO: We should remove this in https://github.com/paritytech/substrate/pull/4686 -/// Currently there is no way to extract this with wasmtime. -const INITIAL_HEAP_PAGES: u32 = 17; - -/// A `WasmRuntime` implementation using the Wasmtime JIT to compile the runtime module to native +/// A `WasmRuntime` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - module: CompiledModule, - context: Context, + module: Module, + imports: Imports, + state_holder: StateHolder, heap_pages: u32, - /// The host functions registered for this instance. host_functions: Vec<&'static dyn Function>, - /// The index of the memory in the module. - memory_index: MemoryIndex, } impl WasmRuntime for WasmtimeRuntime { - fn update_heap_pages(&mut self, heap_pages: u64) -> bool { - self.heap_pages as u64 == heap_pages - } - fn host_functions(&self) -> &[&'static dyn Function] { &self.host_functions } fn call(&mut self, method: &str, data: &[u8]) -> Result> { call_method( - &mut self.context, - &mut self.module, + &self.module, + &mut self.imports, + &self.state_holder, method, data, - self.memory_index, self.heap_pages, ) } + + fn get_global_val(&self, name: &str) -> Result> { + // Yeah, there is no better way currently :( + InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)? + .get_global_val(name) + } } /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to @@ -90,445 +71,105 @@ pub fn create_instance( host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, ) -> std::result::Result { - let heap_pages = u32::try_from(heap_pages) - .map_err(|e| - WasmError::Other(format!("Heap pages can not be converted into `u32`: {:?}", e)) - )?; - - let (compiled_module, context, memory_index) = create_compiled_unit( - code, + // Create the engine, store and finally the module from the given code. + let mut config = Config::new(); + config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize); + + let engine = Engine::new(&config); + let store = Store::new(&engine); + let module = Module::new(&store, code) + .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; + + let state_holder = StateHolder::empty(); + + // Scan all imports, find the matching host functions, and create stubs that adapt arguments + // and results. + let imports = resolve_imports( + &state_holder, + &module, &host_functions, - heap_pages, + heap_pages as u32, allow_missing_func_imports, )?; - let module = compiled_module.module_ref(); - if !module.is_imported_memory(memory_index) { - // Inspect the module for the min and max memory sizes. - let (min_memory_size, max_memory_size) = { - let memory_plan = module.memory_plans - .get(memory_index) - .ok_or_else(|| WasmError::InvalidMemory)?; - (memory_plan.memory.minimum, memory_plan.memory.maximum) - }; - - // Check that heap_pages is within the allowed range. - let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size)); - - if max_heap_pages.map(|m| heap_pages > m).unwrap_or(false) { - return Err(WasmError::InvalidHeapPages) - } - } - Ok(WasmtimeRuntime { - module: compiled_module, - context, - heap_pages, + module, + imports, + state_holder, + heap_pages: heap_pages as u32, host_functions, - memory_index, }) } -#[derive(Debug)] -struct MissingFunction { - name: String, - sig: cranelift_codegen::ir::Signature, -} - -#[derive(Debug)] -struct MissingFunctionStubs { - stubs: HashMap>, -} - -impl MissingFunctionStubs { - fn new() -> Self { - Self { - stubs: HashMap::new(), - } - } - - fn insert(&mut self, module: String, name: String, sig: cranelift_codegen::ir::Signature) { - self.stubs.entry(module).or_insert_with(Vec::new).push(MissingFunction { - name, - sig, - }); - } -} - -fn scan_missing_functions( - code: &[u8], - host_functions: &[&'static dyn Function], -) -> std::result::Result { - let isa = target_isa()?; - let call_conv = isa.default_call_conv(); - - let module = parity_wasm::elements::Module::from_bytes(code) - .map_err(|e| WasmError::Other(format!("cannot deserialize error: {}", e)))?; - - let types = module.type_section().map(|ts| ts.types()).unwrap_or(&[]); - let import_entries = module - .import_section() - .map(|is| is.entries()) - .unwrap_or(&[]); - - let mut missing_functions = MissingFunctionStubs::new(); - for import_entry in import_entries { - let func_ty = match import_entry.external() { - parity_wasm::elements::External::Function(func_ty_idx) => { - let parity_wasm::elements::Type::Function(ref func_ty) = - types.get(*func_ty_idx as usize).ok_or_else(|| { - WasmError::Other(format!("corrupted module, type out of bounds")) - })?; - func_ty - } - _ => { - // We are looking only for missing **functions** here. Any other items, be they - // missing or not, will be handled at the resolution stage later. - continue; - } - }; - let signature = convert_parity_wasm_signature(func_ty); - - if import_entry.module() == "env" { - if let Some(hf) = host_functions - .iter() - .find(|hf| hf.name() == import_entry.field()) - { - if signature == hf.signature() { - continue; - } - } - } - - // This function is either not from the env module, or doesn't have a corresponding host - // function, or the signature is not matching. Add it to the list. - let sig = cranelift_ir_signature(signature, &call_conv); - - missing_functions.insert( - import_entry.module().to_string(), - import_entry.field().to_string(), - sig, - ); - } - - Ok(missing_functions) -} - -fn create_compiled_unit( - code: &[u8], - host_functions: &[&'static dyn Function], - heap_pages: u32, - allow_missing_func_imports: bool, -) -> std::result::Result<(CompiledModule, Context, MemoryIndex), WasmError> { - let compilation_strategy = CompilationStrategy::Cranelift; - - let compiler = new_compiler(compilation_strategy)?; - let mut context = Context::new(Box::new(compiler)); - - // Enable/disable producing of debug info. - context.set_debug_info(false); - - // Instantiate and link the env module. - let global_exports = context.get_global_exports(); - let compiler = new_compiler(compilation_strategy)?; - - let mut missing_functions_stubs = if allow_missing_func_imports { - scan_missing_functions(code, host_functions)? - } else { - // If there are in fact missing functions they will be detected at the instantiation time - // and the module will be rejected. - MissingFunctionStubs::new() - }; - - let env_missing_functions = missing_functions_stubs.stubs - .remove("env") - .unwrap_or_else(|| Vec::new()); - - let (module, memory_index) = instantiate_env_module( - global_exports, - compiler, - host_functions, - heap_pages, - env_missing_functions, - true, - )?; - - context.name_instance("env".to_owned(), module); - - for (module, missing_functions_stubs) in missing_functions_stubs.stubs { - let compiler = new_compiler(compilation_strategy)?; - let global_exports = context.get_global_exports(); - let instance = instantiate_env_module( - global_exports, - compiler, - &[], - heap_pages, - missing_functions_stubs, - false, - )?.0; - context.name_instance(module, instance); - } - - // Compile the wasm module. - let module = context.compile_module(&code) - .map_err(|e| WasmError::Other(format!("module compile error: {}", e)))?; - - Ok((module, context, memory_index.expect("Memory is added on request; qed"))) -} - /// Call a function inside a precompiled Wasm module. fn call_method( - context: &mut Context, - module: &mut CompiledModule, + module: &Module, + imports: &mut Imports, + state_holder: &StateHolder, method: &str, data: &[u8], - memory_index: MemoryIndex, heap_pages: u32, ) -> Result> { - let is_imported_memory = module.module().is_imported_memory(memory_index); - // Old exports get clobbered in `InstanceHandle::new` if we don't explicitly remove them first. - // - // The global exports mechanism is temporary in Wasmtime and expected to be removed. - // https://github.com/CraneStation/wasmtime/issues/332 - clear_globals(&mut *context.get_global_exports().borrow_mut(), is_imported_memory); - - let mut instance = module.instantiate().map_err(|e| Error::Other(e.to_string()))?; - - if !is_imported_memory { - grow_memory(&mut instance, heap_pages)?; - } - - // Initialize the function executor state. - let heap_base = get_heap_base(&instance)?; - let executor_state = FunctionExecutorState::new(heap_base); - reset_env_state_and_take_trap(context, Some(executor_state))?; - - // Write the input data into guest memory. - let (data_ptr, data_len) = inject_input_data(context, &mut instance, data, memory_index)?; - let args = [RuntimeValue::I32(u32::from(data_ptr) as i32), RuntimeValue::I32(data_len as i32)]; - - // Invoke the function in the runtime. - let outcome = context - .invoke(&mut instance, method, &args[..]) - .map_err(|e| Error::Other(format!("error calling runtime: {}", e)))?; - let trap_error = reset_env_state_and_take_trap(context, None)?; - let (output_ptr, output_len) = match outcome { - ActionOutcome::Returned { values } => match values.as_slice() { - [RuntimeValue::I64(retval)] => unpack_ptr_and_len(*retval as u64), - _ => return Err(Error::InvalidReturn), - } - ActionOutcome::Trapped { message } => return Err(trap_error.unwrap_or_else( - || format!("Wasm execution trapped: {}", message).into() - )), - }; - - // Read the output data from guest memory. - let mut output = vec![0; output_len as usize]; - let memory = get_memory_mut(&mut instance, memory_index)?; - read_memory_into(memory, Pointer::new(output_ptr), &mut output)?; - Ok(output) -} - -/// The implementation is based on wasmtime_wasi::instantiate_wasi. -fn instantiate_env_module( - global_exports: Rc>>>, - compiler: Compiler, - host_functions: &[&'static dyn Function], - heap_pages: u32, - missing_functions_stubs: Vec, - add_memory: bool, -) -> std::result::Result<(InstanceHandle, Option), WasmError> { - let isa = target_isa()?; - let pointer_type = isa.pointer_type(); - let call_conv = isa.default_call_conv(); - - let mut fn_builder_ctx = FunctionBuilderContext::new(); - let mut module = Module::new(); - let mut finished_functions = >::new(); - let mut code_memory = CodeMemory::new(); - - for function in host_functions { - let sig = translate_signature( - cranelift_ir_signature(function.signature(), &call_conv), - pointer_type, - ); - let sig_id = module.signatures.push(sig.clone()); - let func_id = module.functions.push(sig_id); - module - .exports - .insert(function.name().to_string(), wasmtime_environ::Export::Function(func_id)); - - let trampoline = make_trampoline( - isa.as_ref(), - &mut code_memory, - &mut fn_builder_ctx, - func_id.index() as u32, - &sig, - )?; - finished_functions.push(trampoline); - } - - for MissingFunction { name, sig } in missing_functions_stubs { - let sig = translate_signature( - sig, - pointer_type, - ); - let sig_id = module.signatures.push(sig.clone()); - let func_id = module.functions.push(sig_id); - module - .exports - .insert(name, wasmtime_environ::Export::Function(func_id)); - let trampoline = make_trampoline( - isa.as_ref(), - &mut code_memory, - &mut fn_builder_ctx, - func_id.index() as u32, - &sig, - )?; - finished_functions.push(trampoline); - } - - code_memory.publish(); - - let memory_id = if add_memory { - let memory = cranelift_wasm::Memory { - minimum: heap_pages + INITIAL_HEAP_PAGES, - maximum: Some(heap_pages + INITIAL_HEAP_PAGES), - shared: false, - }; - let memory_plan = wasmtime_environ::MemoryPlan::for_memory(memory, &Default::default()); - - let memory_id = module.memory_plans.push(memory_plan); - module.exports.insert("memory".into(), wasmtime_environ::Export::Memory(memory_id)); - - Some(memory_id) - } else { - None - }; + let instance_wrapper = InstanceWrapper::new(module, imports, heap_pages)?; + let entrypoint = instance_wrapper.resolve_entrypoint(method)?; + let heap_base = instance_wrapper.extract_heap_base()?; + let allocator = FreeingBumpHeapAllocator::new(heap_base); - let imports = Imports::none(); - let data_initializers = Vec::new(); - let signatures = PrimaryMap::new(); - let env_state = EnvState::new(code_memory, compiler, host_functions); - - let result = InstanceHandle::new( - Rc::new(module), - global_exports, - finished_functions.into_boxed_slice(), - imports, - &data_initializers, - signatures.into_boxed_slice(), - None, - Box::new(env_state), - ); - - result - .map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e))) - .map(|r| (r, memory_id)) -} - -/// Build a new TargetIsa for the host machine. -fn target_isa() -> std::result::Result, WasmError> { - let isa_builder = cranelift_native::builder() - .map_err(|e| WasmError::Other(format!("missing compiler support: {}", e)))?; - let flag_builder = cranelift_codegen::settings::builder(); - Ok(isa_builder.finish(cranelift_codegen::settings::Flags::new(flag_builder))) -} - -fn new_compiler(strategy: CompilationStrategy) -> std::result::Result { - let isa = target_isa()?; - Ok(Compiler::new(isa, strategy)) + perform_call(data, state_holder, instance_wrapper, entrypoint, allocator) } -fn clear_globals(global_exports: &mut HashMap>, is_imported_memory: bool) { - // When memory is imported, we can not delete the global export. - if !is_imported_memory { - global_exports.remove("memory"); - } - global_exports.remove("__heap_base"); - global_exports.remove("__indirect_function_table"); -} - -fn grow_memory(instance: &mut InstanceHandle, pages: u32) -> Result<()> { - // This is safe to wrap in an unsafe block as: - // - The result of the `lookup_immutable` call is not mutated - // - The definition pointer is returned by a lookup on a valid instance - let memory_index = unsafe { - match instance.lookup_immutable("memory") { - Some(Export::Memory { definition, vmctx: _, memory: _ }) => - instance.memory_index(&*definition), - _ => return Err(Error::InvalidMemoryReference), +fn perform_call( + data: &[u8], + state_holder: &StateHolder, + instance_wrapper: InstanceWrapper, + entrypoint: wasmtime::Func, + mut allocator: FreeingBumpHeapAllocator, +) -> Result> { + let (data_ptr, data_len) = inject_input_data(&instance_wrapper, &mut allocator, data)?; + + let host_state = HostState::new(allocator, instance_wrapper); + let (ret, host_state) = state_holder.with_initialized_state(host_state, || { + match entrypoint.call(&[ + wasmtime::Val::I32(u32::from(data_ptr) as i32), + wasmtime::Val::I32(u32::from(data_len) as i32), + ]) { + Ok(results) => { + let retval = results[0].unwrap_i64() as u64; + Ok(unpack_ptr_and_len(retval)) + } + Err(trap) => { + return Err(Error::from(format!( + "Wasm execution trapped: {}", + trap.message() + ))); + } } - }; - instance.memory_grow(memory_index, pages) - .map(|_| ()) - .ok_or_else(|| "requested heap_pages would exceed maximum memory size".into()) -} + }); + let (output_ptr, output_len) = ret?; -fn get_env_state(context: &mut Context) -> Result<&mut EnvState> { - let env_instance = context.get_instance("env") - .map_err(|err| format!("cannot find \"env\" module: {}", err))?; - env_instance - .host_state() - .downcast_mut::() - .ok_or_else(|| "cannot get \"env\" module host state".into()) -} + let instance = host_state.into_instance(); + let output = extract_output_data(&instance, output_ptr, output_len)?; -fn reset_env_state_and_take_trap( - context: &mut Context, - executor_state: Option, -) -> Result> -{ - let env_state = get_env_state(context)?; - env_state.executor_state = executor_state; - Ok(env_state.take_trap()) + Ok(output) } fn inject_input_data( - context: &mut Context, - instance: &mut InstanceHandle, + instance: &InstanceWrapper, + allocator: &mut FreeingBumpHeapAllocator, data: &[u8], - memory_index: MemoryIndex, ) -> Result<(Pointer, WordSize)> { - let env_state = get_env_state(context)?; - let executor_state = env_state.executor_state - .as_mut() - .ok_or_else(|| "cannot get \"env\" module executor state")?; - - let memory = get_memory_mut(instance, memory_index)?; - let data_len = data.len() as WordSize; - let data_ptr = executor_state.heap().allocate(memory, data_len)?; - write_memory_from(memory, data_ptr, data)?; + let data_ptr = instance.allocate(allocator, data_len)?; + instance.write_memory_from(data_ptr, data)?; Ok((data_ptr, data_len)) } -fn get_memory_mut(instance: &mut InstanceHandle, memory_index: MemoryIndex) -> Result<&mut [u8]> { - match instance.lookup_by_declaration(&wasmtime_environ::Export::Memory(memory_index)) { - // This is safe to wrap in an unsafe block as: - // - The definition pointer is returned by a lookup on a valid instance and thus points to - // a valid memory definition - Export::Memory { definition, vmctx: _, memory: _ } => unsafe { - Ok(std::slice::from_raw_parts_mut( - (*definition).base, - (*definition).current_length, - )) - }, - _ => Err(Error::InvalidMemoryReference), - } -} - -fn get_heap_base(instance: &InstanceHandle) -> Result { - // This is safe to wrap in an unsafe block as: - // - The result of the `lookup_immutable` call is not mutated - // - The definition pointer is returned by a lookup on a valid instance - // - The defined value is checked to be an I32, which can be read safely as a u32 - unsafe { - match instance.lookup_immutable("__heap_base") { - Some(Export::Global { definition, vmctx: _, global }) - if global.ty == ir::types::I32 => - Ok(*(*definition).as_u32()), - _ => return Err(Error::HeapBaseNotFoundOrInvalid), - } - } +fn extract_output_data( + instance: &InstanceWrapper, + output_ptr: u32, + output_len: u32, +) -> Result> { + let mut output = vec![0; output_len as usize]; + instance.read_memory_into(Pointer::new(output_ptr), &mut output)?; + Ok(output) } diff --git a/client/executor/wasmtime/src/state_holder.rs b/client/executor/wasmtime/src/state_holder.rs new file mode 100644 index 0000000000000000000000000000000000000000..57564ed3ec414506e1960c41654e42be96e44576 --- /dev/null +++ b/client/executor/wasmtime/src/state_holder.rs @@ -0,0 +1,77 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::host::{HostContext, HostState}; +use std::cell::RefCell; +use std::rc::Rc; + +/// A common place to store a reference to the `HostState`. +/// +/// This structure is passed into each host function handler and retained in the implementation of +/// `WasmRuntime`. Whenever a call into a runtime method is initiated, the host state is populated +/// with the state for that runtime method call. +/// +/// During the execution of the runtime method call, wasm can call imported host functions. When +/// that happens the host function handler gets a `HostContext` (obtainable through having a +/// `HostState` reference). +#[derive(Clone)] +pub struct StateHolder { + // This is `Some` only during a call. + state: Rc>>, +} + +impl StateHolder { + /// Create a placeholder `StateHolder`. + pub fn empty() -> StateHolder { + StateHolder { + state: Rc::new(RefCell::new(None)), + } + } + + /// Provide `HostState` for the runtime method call and execute the given function `f`. + /// + /// During the execution of the provided function `with_context` will be callable. + pub fn with_initialized_state(&self, state: HostState, f: F) -> (R, HostState) + where + F: FnOnce() -> R, + { + *self.state.borrow_mut() = Some(state); + + let ret = f(); + let state = self + .state + .borrow_mut() + .take() + .expect("cannot be None since was just assigned; qed"); + + (ret, state) + } + + /// Create a `HostContext` from the contained `HostState` and execute the given function `f`. + /// + /// This function is only callable within closure passed to `init_state`. Otherwise, the passed + /// context will be `None`. + pub fn with_context(&self, f: F) -> R + where + F: FnOnce(Option) -> R, + { + let state = self.state.borrow(); + match *state { + Some(ref state) => f(Some(state.materialize())), + None => f(None), + } + } +} diff --git a/client/executor/wasmtime/src/trampoline.rs b/client/executor/wasmtime/src/trampoline.rs deleted file mode 100644 index 8a2147760921b62d55852d03de1562c000053148..0000000000000000000000000000000000000000 --- a/client/executor/wasmtime/src/trampoline.rs +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! The trampoline is the dynamically generated entry point to a runtime host call. -//! -//! This code is based on and large parts are copied from wasmtime's -//! wasmtime-api/src/trampoline/func.rs. - -use crate::function_executor::{FunctionExecutorState, FunctionExecutor}; -use sc_executor_common::error::{Error, WasmError}; - -use cranelift_codegen::{Context, binemit, ir, isa}; -use cranelift_codegen::ir::{InstBuilder, StackSlotData, StackSlotKind, TrapCode}; -use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; -use cranelift_codegen::print_errors::pretty_error; -use wasmtime_jit::{CodeMemory, Compiler}; -use wasmtime_environ::CompiledFunction; -use wasmtime_runtime::{VMContext, VMFunctionBody}; -use sp_wasm_interface::{Function, Value, ValueType}; -use std::{cmp, panic::{self, AssertUnwindSafe}, ptr}; - -const CALL_SUCCESS: u32 = 0; -const CALL_FAILED_WITH_ERROR: u32 = 1; -const CALL_WITH_BAD_HOST_STATE: u32 = 2; - -/// A code to trap with that indicates a host call error. -const TRAP_USER_CODE: u16 = 0; - -/// The only Wasm types allowed in host function signatures (I32, I64, F32, F64) are all -/// represented in at most 8 bytes. -const MAX_WASM_TYPE_SIZE: usize = 8; - -/// The top-level host state of the "env" module. This state is used by the trampoline function to -/// construct a `FunctionExecutor` which can execute the host call. -pub struct EnvState { - host_functions: Vec<&'static dyn Function>, - compiler: Compiler, - // The code memory must be kept around on the state to prevent it from being dropped. - #[allow(dead_code)] - code_memory: CodeMemory, - trap: Option, - /// The executor state stored across host calls during a single Wasm runtime call. - /// During a runtime call, this MUST be `Some`. - pub executor_state: Option, -} - -impl EnvState { - /// Construct a new `EnvState` which owns the given code memory. - pub fn new( - code_memory: CodeMemory, - compiler: Compiler, - host_functions: &[&'static dyn Function], - ) -> Self { - EnvState { - trap: None, - compiler, - code_memory, - executor_state: None, - host_functions: host_functions.to_vec(), - } - } - - /// Resets the trap error to None and returns the current value. - pub fn take_trap(&mut self) -> Option { - self.trap.take() - } -} - -/// This is called by the dynamically generated trampoline taking the function index and reference -/// to the call arguments on the stack as arguments. Returns zero on success and a non-zero value -/// on failure. -unsafe extern "C" fn stub_fn(vmctx: *mut VMContext, func_index: u32, values_vec: *mut i64) -> u32 { - if let Some(state) = (*vmctx).host_state().downcast_mut::() { - match stub_fn_inner( - vmctx, - &state.host_functions, - &mut state.compiler, - state.executor_state.as_mut(), - func_index, - values_vec, - ) { - Ok(()) => CALL_SUCCESS, - Err(err) => { - state.trap = Some(err); - CALL_FAILED_WITH_ERROR - } - } - } else { - // Well, we can't even set a trap message, so we'll just exit without one. - CALL_WITH_BAD_HOST_STATE - } -} - -/// Implements most of the logic in `stub_fn` but returning a `Result` instead of an integer error -/// for the sake of readability. -unsafe fn stub_fn_inner( - vmctx: *mut VMContext, - externals: &[&dyn Function], - compiler: &mut Compiler, - executor_state: Option<&mut FunctionExecutorState>, - func_index: u32, - values_vec: *mut i64, -) -> Result<(), Error> { - let func = externals.get(func_index as usize) - .ok_or_else(|| format!("call to undefined external function with index {}", func_index))?; - let executor_state = executor_state - .ok_or_else(|| "executor state is None during call to external function")?; - - // Build the external function context. - let mut context = FunctionExecutor::new(vmctx, compiler, executor_state)?; - let mut context = AssertUnwindSafe(&mut context); - - // Execute and write output back to the stack. - let return_val = panic::catch_unwind(move || { - let signature = func.signature(); - - // Read the arguments from the stack. - let mut args = signature.args.iter() - .enumerate() - .map(|(i, ¶m_type)| read_value_from(values_vec.offset(i as isize), param_type)); - - func.execute(&mut **context, &mut args) - }); - - match return_val { - Ok(ret_val) => { - if let Some(val) = ret_val - .map_err(|e| Error::FunctionExecution(func.name().to_string(), e))? { - write_value_to(values_vec, val); - } - - Ok(()) - }, - Err(e) => { - let message = if let Some(err) = e.downcast_ref::() { - err.to_string() - } else if let Some(err) = e.downcast_ref::<&str>() { - err.to_string() - } else { - "Panicked without any further information!".into() - }; - - Err(Error::FunctionExecution(func.name().to_string(), message)) - } - } -} - -/// Create a trampoline for invoking a host function. -/// -/// The trampoline is a dynamically generated entry point to a runtime host call. The function is -/// generated by manually constructing Cranelift IR and using the Cranelift compiler. The -/// trampoline embeds the function index as a constant and delegates to a stub function in Rust, -/// which takes the function index and a memory reference to the stack arguments and return value -/// slots. -/// -/// This code is of modified copy of wasmtime's wasmtime-api/src/trampoline/func.rs. -pub fn make_trampoline( - isa: &dyn isa::TargetIsa, - code_memory: &mut CodeMemory, - fn_builder_ctx: &mut FunctionBuilderContext, - func_index: u32, - signature: &ir::Signature, -) -> Result<*const VMFunctionBody, WasmError> { - // Mostly reverse copy of the similar method from wasmtime's - // wasmtime-jit/src/compiler.rs. - let pointer_type = isa.pointer_type(); - let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Ensure that the first parameter of the generated function is the `VMContext` pointer. - assert_eq!( - signature.params[0], - ir::AbiParam::special(pointer_type, ir::ArgumentPurpose::VMContext) - ); - - // Add the `vmctx` parameter. - stub_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the `func_index` parameter. - stub_sig.params.push(ir::AbiParam::new(ir::types::I32)); - - // Add the `values_vec` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); - - // Add error/trap return. - stub_sig.returns.push(ir::AbiParam::new(ir::types::I32)); - - // Each parameter and return value gets a 64-bit (8-byte) wide slot on the stack, as that is - // large enough to fit all Wasm primitive types that can be used in host function signatures. - // The `VMContext` pointer, which is a parameter of the function signature, is excluded as it - // is passed directly to the stub function rather than being looked up on the caller stack from - // the `values_vec` pointer. - let values_vec_len = cmp::max(signature.params.len() - 1, signature.returns.len()); - let values_vec_size = (MAX_WASM_TYPE_SIZE * values_vec_len) as u32; - - let mut context = Context::new(); - context.func = - ir::Function::with_name_signature(ir::ExternalName::user(0, 0), signature.clone()); - - let ss = context.func.create_stack_slot(StackSlotData::new( - StackSlotKind::ExplicitSlot, - values_vec_size, - )); - - { - let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); - let block0 = builder.create_ebb(); - - builder.append_ebb_params_for_function_params(block0); - builder.switch_to_block(block0); - builder.seal_block(block0); - - let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); - let mflags = ir::MemFlags::trusted(); - for i in 1..signature.params.len() { - let val = builder.func.dfg.ebb_params(block0)[i]; - builder.ins().store( - mflags, - val, - values_vec_ptr_val, - ((i - 1) * MAX_WASM_TYPE_SIZE) as i32, - ); - } - - let vmctx_ptr_val = builder.func.dfg.ebb_params(block0)[0]; - let func_index_val = builder.ins().iconst(ir::types::I32, func_index as i64); - - let callee_args = vec![vmctx_ptr_val, func_index_val, values_vec_ptr_val]; - - let new_sig = builder.import_signature(stub_sig.clone()); - - let callee_value = builder - .ins() - .iconst(pointer_type, stub_fn as *const VMFunctionBody as i64); - let call = builder - .ins() - .call_indirect(new_sig, callee_value, &callee_args); - - let call_result = builder.func.dfg.inst_results(call)[0]; - builder.ins().trapnz(call_result, TrapCode::User(TRAP_USER_CODE)); - - let mflags = ir::MemFlags::trusted(); - let mut results = Vec::new(); - for (i, r) in signature.returns.iter().enumerate() { - let load = builder.ins().load( - r.value_type, - mflags, - values_vec_ptr_val, - (i * MAX_WASM_TYPE_SIZE) as i32, - ); - results.push(load); - } - builder.ins().return_(&results); - builder.finalize() - } - - let mut code_buf: Vec = Vec::new(); - let mut reloc_sink = RelocSink; - let mut trap_sink = binemit::NullTrapSink {}; - let mut stackmap_sink = binemit::NullStackmapSink {}; - context - .compile_and_emit( - isa, - &mut code_buf, - &mut reloc_sink, - &mut trap_sink, - &mut stackmap_sink, - ) - .map_err(|e| { - WasmError::Instantiation(format!( - "failed to compile trampoline: {}", - pretty_error(&context.func, Some(isa), e) - )) - })?; - - let mut unwind_info = Vec::new(); - context.emit_unwind_info(isa, &mut unwind_info); - - let func_ref = code_memory - .allocate_for_function(&CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - }) - .map_err(|e| WasmError::Instantiation(format!("failed to allocate code memory: {}", e)))?; - - Ok(func_ref.as_ptr()) -} - -/// We don't expect trampoline compilation to produce any relocations, so -/// this `RelocSink` just asserts that it doesn't recieve any. -struct RelocSink; - -impl binemit::RelocSink for RelocSink { - fn reloc_ebb( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _ebb_offset: binemit::CodeOffset, - ) { - panic!("trampoline compilation should not produce ebb relocs"); - } - fn reloc_external( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _name: &ir::ExternalName, - _addend: binemit::Addend, - ) { - panic!("trampoline compilation should not produce external symbol relocs"); - } - fn reloc_constant( - &mut self, - _code_offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _constant_offset: ir::ConstantOffset, - ) { - panic!("trampoline compilation should not produce constant relocs"); - } - fn reloc_jt( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _jt: ir::JumpTable, - ) { - panic!("trampoline compilation should not produce jump table relocs"); - } -} - -unsafe fn write_value_to(p: *mut i64, val: Value) { - match val { - Value::I32(i) => ptr::write(p as *mut i32, i), - Value::I64(i) => ptr::write(p as *mut i64, i), - Value::F32(u) => ptr::write(p as *mut u32, u), - Value::F64(u) => ptr::write(p as *mut u64, u), - } -} - -unsafe fn read_value_from(p: *const i64, ty: ValueType) -> Value { - match ty { - ValueType::I32 => Value::I32(ptr::read(p as *const i32)), - ValueType::I64 => Value::I64(ptr::read(p as *const i64)), - ValueType::F32 => Value::F32(ptr::read(p as *const u32)), - ValueType::F64 => Value::F64(ptr::read(p as *const u64)), - } -} diff --git a/client/executor/wasmtime/src/util.rs b/client/executor/wasmtime/src/util.rs index 77fb8af3867a4dd835eb496c67cae7914aaa7a8e..d2de95d4cc7150d6ed8768ad48181ae9f9885cb4 100644 --- a/client/executor/wasmtime/src/util.rs +++ b/client/executor/wasmtime/src/util.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,31 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_executor_common::error::{Error, Result}; - -use cranelift_codegen::{ir, isa}; use std::ops::Range; -use sp_wasm_interface::{Pointer, Signature, ValueType}; - -/// Read data from a slice of memory into a destination buffer. -/// -/// Returns an error if the read would go out of the memory bounds. -pub fn read_memory_into(memory: &[u8], address: Pointer, dest: &mut [u8]) -> Result<()> { - let range = checked_range(address.into(), dest.len(), memory.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - dest.copy_from_slice(&memory[range]); - Ok(()) -} - -/// Write data to a slice of memory. -/// -/// Returns an error if the write would go out of the memory bounds. -pub fn write_memory_from(memory: &mut [u8], address: Pointer, data: &[u8]) -> Result<()> { - let range = checked_range(address.into(), data.len(), memory.len()) - .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; - &mut memory[range].copy_from_slice(data); - Ok(()) -} /// Construct a range from an offset to a data length after the offset. /// Returns None if the end of the range would exceed some maximum offset. @@ -50,82 +26,3 @@ pub fn checked_range(offset: usize, len: usize, max: usize) -> Option Signature { - fn convert_value_type(val_ty: parity_wasm::elements::ValueType) -> ValueType { - use parity_wasm::elements::ValueType::*; - match val_ty { - I32 => ValueType::I32, - I64 => ValueType::I64, - F32 => ValueType::F32, - F64 => ValueType::F64, - } - } - - Signature::new( - func_ty.params().iter().cloned().map(convert_value_type).collect::>(), - func_ty.return_type().map(convert_value_type), - ) -} - -/// Convert a wasm_interface Signature into a cranelift_codegen Signature. -pub fn cranelift_ir_signature(signature: Signature, call_conv: &isa::CallConv) -> ir::Signature { - ir::Signature { - params: signature.args.iter() - .map(cranelift_ir_type) - .map(ir::AbiParam::new) - .collect(), - returns: signature.return_value.iter() - .map(cranelift_ir_type) - .map(ir::AbiParam::new) - .collect(), - call_conv: call_conv.clone(), - } -} - -/// Convert a wasm_interface ValueType into a cranelift_codegen Type. -pub fn cranelift_ir_type(value_type: &ValueType) -> ir::types::Type { - match value_type { - ValueType::I32 => ir::types::I32, - ValueType::I64 => ir::types::I64, - ValueType::F32 => ir::types::F32, - ValueType::F64 => ir::types::F64, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use assert_matches::assert_matches; - - #[test] - fn test_read_memory_into() { - let mut memory = [0; 20]; - let mut dest = [0; 5]; - - &mut memory[15..20].copy_from_slice(b"hello"); - - read_memory_into(&memory[..], Pointer::new(15), &mut dest[..]).unwrap(); - - // Test that out of bounds read fails. - assert_matches!( - read_memory_into(&memory[..], Pointer::new(16), &mut dest[..]), - Err(Error::Other(_)) - ) - } - - #[test] - fn test_write_memory_from() { - let mut memory = [0; 20]; - let data = b"hello"; - - write_memory_from(&mut memory[..], Pointer::new(15), data).unwrap(); - - // Test that out of bounds write fails. - assert_matches!( - write_memory_from(&mut memory[..], Pointer::new(16), data), - Err(Error::Other(_)) - ) - } -} diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 1249bff751d521a99c1a9ca00278aa90f2e8b416..733e5415b82a75f17cecafe2c82d95ccc8dbdea6 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -1,47 +1,49 @@ [package] name = "sc-finality-grandpa" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-alpha.1", path = "../../utils/fork-tree" } futures = "0.3.1" -futures-timer = "2.0.2" +futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" assert_matches = "1.3.0" parity-scale-codec = { version = "1.0.0", features = ["derive"] } -sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sc-keystore = { version = "2.0.0", path = "../keystore" } +sp-arithmetic = { version = "2.0.0-alpha.1", path = "../../primitives/arithmetic" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../keystore" } serde_json = "1.0.41" -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-inherents = { version = "2.0.0", path = "../../primitives/inherents" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-network = { version = "0.8", path = "../network" } -sc-network-gossip = { version = "0.8", path = "../network-gossip" } -sp-finality-tracker = { version = "2.0.0", path = "../../primitives/finality-tracker" } -sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sp-inherents = { version = "2.0.0-alpha.1", path = "../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sc-network-gossip = { version = "0.8.0-alpha.1", path = "../network-gossip" } +sp-finality-tracker = { version = "2.0.0-alpha.1", path = "../../primitives/finality-tracker" } +sp-finality-grandpa = { version = "2.0.0-alpha.1", path = "../../primitives/finality-grandpa" } finality-grandpa = { version = "0.11.1", features = ["derive-codec"] } pin-project = "0.4.6" [dev-dependencies] finality-grandpa = { version = "0.11.1", features = ["derive-codec", "test-helpers"] } -sc-network = { version = "0.8", path = "../network" } -sc-network-test = { version = "0.8.0", path = "../network/test" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sc-network-test = { version = "0.8.0-dev", path = "../network/test" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/babe" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } env_logger = "0.7.0" tokio = "0.1.22" tempfile = "3.1.0" -sp-api = { version = "2.0.0", path = "../../primitives/api" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } futures01 = { package = "futures", version = "0.1.29" } diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 227ecaa3707eb5e582b2a380d936eb3d4e40648d..1f4e22359712fa2f398bd846e1fbe1620a47efa4 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -65,6 +65,7 @@ mod periodic; pub(crate) mod tests; pub use sp_finality_grandpa::GRANDPA_ENGINE_ID; +pub const GRANDPA_PROTOCOL_NAME: &[u8] = b"/paritytech/grandpa/1"; // cost scalars for reporting peers. mod cost { @@ -74,7 +75,7 @@ mod cost { pub(super) const MALFORMED_CATCH_UP: Rep = Rep::new(-1000, "Grandpa: Malformed cath-up"); pub(super) const MALFORMED_COMMIT: Rep = Rep::new(-1000, "Grandpa: Malformed commit"); pub(super) const FUTURE_MESSAGE: Rep = Rep::new(-500, "Grandpa: Future message"); - pub(super) const UNKNOWN_VOTER: Rep = Rep::new(-150, "Grandpa: Uknown voter"); + pub(super) const UNKNOWN_VOTER: Rep = Rep::new(-150, "Grandpa: Unknown voter"); pub(super) const INVALID_VIEW_CHANGE: Rep = Rep::new(-500, "Grandpa: Invalid view change"); pub(super) const PER_UNDECODABLE_BYTE: i32 = -5; @@ -83,7 +84,7 @@ mod cost { pub(super) const INVALID_CATCH_UP: Rep = Rep::new(-5000, "Grandpa: Invalid catch-up"); pub(super) const INVALID_COMMIT: Rep = Rep::new(-5000, "Grandpa: Invalid commit"); pub(super) const OUT_OF_SCOPE_MESSAGE: Rep = Rep::new(-500, "Grandpa: Out-of-scope message"); - pub(super) const CATCH_UP_REQUEST_TIMEOUT: Rep = Rep::new(-200, "Grandpa: Catch-up reqeust timeout"); + pub(super) const CATCH_UP_REQUEST_TIMEOUT: Rep = Rep::new(-200, "Grandpa: Catch-up request timeout"); // cost of answering a catch up request pub(super) const CATCH_UP_REPLY: Rep = Rep::new(-200, "Grandpa: Catch-up reply"); @@ -119,9 +120,8 @@ pub trait Network: GossipNetwork + Clone + Send + 'static fn set_sync_fork_request(&self, peers: Vec, hash: Block::Hash, number: NumberFor); } -impl Network for Arc> where +impl Network for Arc> where B: BlockT, - S: sc_network::specialization::NetworkSpecialization, H: sc_network::ExHashT, { fn set_sync_fork_request(&self, peers: Vec, hash: B::Hash, number: NumberFor) { @@ -153,14 +153,14 @@ pub(crate) struct NetworkBridge> { /// `NeighborPacketWorker` processing packets sent through the `NeighborPacketSender`. // - // `NetworkBridge` is required to be clonable, thus one needs to be able to clone its children, - // thus one has to wrap neighor_packet_worker with an `Arc` `Mutex`. + // `NetworkBridge` is required to be cloneable, thus one needs to be able to clone its children, + // thus one has to wrap `neighbor_packet_worker` with an `Arc` `Mutex`. neighbor_packet_worker: Arc>>, /// Receiver side of the peer report stream populated by the gossip validator, forwarded to the /// gossip engine. // - // `NetworkBridge` is required to be clonable, thus one needs to be able to clone its children, + // `NetworkBridge` is required to be cloneable, thus one needs to be able to clone its children, // thus one has to wrap gossip_validator_report_stream with an `Arc` `Mutex`. Given that it is // just an `UnboundedReceiver`, one could also switch to a multi-producer-*multi*-consumer // channel implementation. @@ -178,7 +178,6 @@ impl> NetworkBridge { service: N, config: crate::Config, set_state: crate::environment::SharedVoterSetState, - executor: &impl futures::task::Spawn, ) -> Self { let (validator, report_stream) = GossipValidator::new( config, @@ -186,7 +185,12 @@ impl> NetworkBridge { ); let validator = Arc::new(validator); - let gossip_engine = GossipEngine::new(service.clone(), executor, GRANDPA_ENGINE_ID, validator.clone()); + let gossip_engine = GossipEngine::new( + service.clone(), + GRANDPA_ENGINE_ID, + GRANDPA_PROTOCOL_NAME, + validator.clone() + ); { // register all previous votes with the gossip service so that they're @@ -374,10 +378,9 @@ impl> NetworkBridge { |to, neighbor| self.neighbor_sender.send(to, neighbor), ); - let service = self.gossip_engine.clone(); let topic = global_topic::(set_id.0); let incoming = incoming_global( - service, + self.gossip_engine.clone(), topic, voters, self.validator.clone(), @@ -419,7 +422,7 @@ impl> NetworkBridge { impl> Future for NetworkBridge { type Output = Result<(), Error>; - fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { loop { match self.neighbor_packet_worker.lock().poll_next_unpin(cx) { Poll::Ready(Some((to, packet))) => { @@ -444,6 +447,12 @@ impl> Future for NetworkBridge { } } + match self.gossip_engine.poll_unpin(cx) { + // The gossip engine future finished. We should do the same. + Poll::Ready(()) => return Poll::Ready(Ok(())), + Poll::Pending => {}, + } + Poll::Pending } } diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index 463589969012ff7c3920a0562e065cef391ffc85..f2e79e8f14486e5bb0951acfd701e44aaec5a28c 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -19,7 +19,7 @@ use futures_timer::Delay; use futures::{channel::mpsc, future::{FutureExt as _}, prelude::*, ready, stream::Stream}; use log::debug; -use std::{pin::Pin, task::{Context, Poll}, time::{Instant, Duration}}; +use std::{pin::Pin, task::{Context, Poll}, time::Duration}; use sc_network::PeerId; use sp_runtime::traits::{NumberFor, Block as BlockT}; @@ -28,10 +28,6 @@ use super::gossip::{NeighborPacket, GossipMessage}; // How often to rebroadcast, in cases where no new packets are created. const REBROADCAST_AFTER: Duration = Duration::from_secs(2 * 60); -fn rebroadcast_instant() -> Instant { - Instant::now() + REBROADCAST_AFTER -} - /// A sender used to send neighbor packets to a background job. #[derive(Clone)] pub(super) struct NeighborPacketSender( @@ -85,7 +81,7 @@ impl Stream for NeighborPacketWorker { match this.rx.poll_next_unpin(cx) { Poll::Ready(None) => return Poll::Ready(None), Poll::Ready(Some((to, packet))) => { - this.delay.reset(rebroadcast_instant()); + this.delay.reset(REBROADCAST_AFTER); this.last = Some((to.clone(), packet.clone())); return Poll::Ready(Some((to, GossipMessage::::from(packet.clone())))); @@ -98,7 +94,7 @@ impl Stream for NeighborPacketWorker { // Getting this far here implies that the timer fired. - this.delay.reset(rebroadcast_instant()); + this.delay.reset(REBROADCAST_AFTER); // Make sure the underlying task is scheduled for wake-up. // diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index e10d24a16a264f9f75b9a4ea6b745f819eebfa91..96761a2f3c07c567aaefdc3f0540dee3c0cabb1c 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use sp_keyring::Ed25519Keyring; use parity_scale_codec::Encode; use sp_runtime::{ConsensusEngineId, traits::NumberFor}; -use std::{pin::Pin, task::{Context, Poll}}; +use std::{borrow::Cow, pin::Pin, task::{Context, Poll}}; use crate::environment::SharedVoterSetState; use sp_finality_grandpa::{AuthorityList, GRANDPA_ENGINE_ID}; use super::gossip::{self, GossipValidator}; @@ -44,20 +44,12 @@ pub(crate) struct TestNetwork { sender: mpsc::UnboundedSender, } -impl TestNetwork { - fn event_stream_03(&self) -> Pin + Send>> { +impl sc_network_gossip::Network for TestNetwork { + fn event_stream(&self) -> Pin + Send>> { let (tx, rx) = mpsc::unbounded(); let _ = self.sender.unbounded_send(Event::EventStream(tx)); Box::pin(rx) } -} - -impl sc_network_gossip::Network for TestNetwork { - fn event_stream(&self) -> Box + Send> { - Box::new( - self.event_stream_03().map(Ok::<_, ()>).compat() - ) - } fn report_peer(&self, who: sc_network::PeerId, cost_benefit: sc_network::ReputationChange) { let _ = self.sender.unbounded_send(Event::Report(who, cost_benefit)); @@ -69,7 +61,7 @@ impl sc_network_gossip::Network for TestNetwork { let _ = self.sender.unbounded_send(Event::WriteNotification(who, message)); } - fn register_notifications_protocol(&self, _: ConsensusEngineId) {} + fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {} fn announce(&self, block: Hash, _associated_data: Vec) { let _ = self.sender.unbounded_send(Event::Announce(block)); @@ -165,7 +157,7 @@ fn voter_set_state() -> SharedVoterSetState { } // needs to run in a tokio runtime. -pub(crate) fn make_test_network(executor: &impl futures::task::Spawn) -> ( +pub(crate) fn make_test_network() -> ( impl Future, TestNetwork, ) { @@ -187,7 +179,6 @@ pub(crate) fn make_test_network(executor: &impl futures::task::Spawn) -> ( net.clone(), config(), voter_set_state(), - executor, ); ( @@ -261,8 +252,7 @@ fn good_commit_leads_to_relay() { let id = sc_network::PeerId::random(); let global_topic = super::global_topic::(set_id); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let test = make_test_network(&threads_pool).0 + let test = make_test_network().0 .then(move |tester| { // register a peer. tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); @@ -281,6 +271,7 @@ fn good_commit_leads_to_relay() { } let commit_to_send = encoded_commit.clone(); + let network_bridge = tester.net_handle.clone(); // asking for global communication will cause the test network // to send us an event asking us for a stream. use it to @@ -325,7 +316,7 @@ fn good_commit_leads_to_relay() { // once the message is sent and commit is "handled" we should have // a repropagation event coming from the network. - future::join(send_message, handle_commit).then(move |(tester, ())| { + let fut = future::join(send_message, handle_commit).then(move |(tester, ())| { tester.filter_network_events(move |event| match event { Event::WriteNotification(_, data) => { data == encoded_commit @@ -333,7 +324,11 @@ fn good_commit_leads_to_relay() { _ => false, }) }) - .map(|_| ()) + .map(|_| ()); + + // Poll both the future sending and handling the commit, as well as the underlying + // NetworkBridge. Complete once the former completes. + future::select(fut, network_bridge) }); futures::executor::block_on(test); @@ -385,8 +380,7 @@ fn bad_commit_leads_to_report() { let id = sc_network::PeerId::random(); let global_topic = super::global_topic::(set_id); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let test = make_test_network(&threads_pool).0 + let test = make_test_network().0 .map(move |tester| { // register a peer. tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); @@ -405,6 +399,7 @@ fn bad_commit_leads_to_report() { } let commit_to_send = encoded_commit.clone(); + let network_bridge = tester.net_handle.clone(); // asking for global communication will cause the test network // to send us an event asking us for a stream. use it to @@ -427,7 +422,7 @@ fn bad_commit_leads_to_report() { _ => false, }); - // when the commit comes in, we'll tell the callback it was good. + // when the commit comes in, we'll tell the callback it was bad. let handle_commit = commits_in.into_future() .map(|(item, _)| { match item.unwrap() { @@ -440,7 +435,7 @@ fn bad_commit_leads_to_report() { // once the message is sent and commit is "handled" we should have // a report event coming from the network. - future::join(send_message, handle_commit).then(move |(tester, ())| { + let fut = future::join(send_message, handle_commit).then(move |(tester, ())| { tester.filter_network_events(move |event| match event { Event::Report(who, cost_benefit) => { who == id && cost_benefit == super::cost::INVALID_COMMIT @@ -448,7 +443,11 @@ fn bad_commit_leads_to_report() { _ => false, }) }) - .map(|_| ()) + .map(|_| ()); + + // Poll both the future sending and handling the commit, as well as the underlying + // NetworkBridge. Complete once the former completes. + future::select(fut, network_bridge) }); futures::executor::block_on(test); @@ -458,8 +457,7 @@ fn bad_commit_leads_to_report() { fn peer_with_higher_view_leads_to_catch_up_request() { let id = sc_network::PeerId::random(); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let (tester, mut net) = make_test_network(&threads_pool); + let (tester, mut net) = make_test_network(); let test = tester .map(move |tester| { // register a peer with authority role. diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index fb0f7fd4a9bab66e91447209c7457d1e7ff3aca8..d52db6c099859af1d38690d33104f9f8a063dd4c 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -154,7 +154,7 @@ impl FinalityProofProvider } } -impl sc_network::FinalityProofProvider for FinalityProofProvider +impl sc_network::config::FinalityProofProvider for FinalityProofProvider where Block: BlockT, NumberFor: BlockNumberOps, @@ -920,7 +920,7 @@ pub(crate) mod tests { } #[test] - fn finality_proof_check_fails_when_intemediate_fragment_has_unknown_headers() { + fn finality_proof_check_fails_when_intermediate_fragment_has_unknown_headers() { let blockchain = test_blockchain(); // when intermediate (#0) fragment has non-empty unknown headers @@ -945,7 +945,7 @@ pub(crate) mod tests { } #[test] - fn finality_proof_check_fails_when_intemediate_fragment_has_no_authorities_proof() { + fn finality_proof_check_fails_when_intermediate_fragment_has_no_authorities_proof() { let blockchain = test_blockchain(); // when intermediate (#0) fragment has empty authorities proof @@ -1004,7 +1004,7 @@ pub(crate) mod tests { #[test] fn finality_proof_is_none_if_first_justification_is_generated_by_unknown_set() { // this is the case for forced change: set_id has been forcibly increased on full node - // and ligh node missed that + // and light node missed that // => justification verification will fail on light node anyways, so we do not return // finality proof at all let blockchain = test_blockchain(); diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 64fc62bfe7a8bd4d46c94fd2c0904e5b862ad989..2eb1b4a7c889c3929c57ac3149e373a6c86c736f 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -398,7 +398,7 @@ impl BlockImport mut block: BlockImportParams, new_cache: HashMap>, ) -> Result { - let hash = block.post_header().hash(); + let hash = block.post_hash(); let number = block.header.number().clone(); // early exit if block already in chain, otherwise the check for diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index afca0ed48b5e395223d696f1db001dbcc5185349..650b59dfff662a513836d5cf53d30e9ee9c5d3ce 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -96,7 +96,6 @@ mod voting_rule; pub use finality_proof::FinalityProofProvider; pub use justification::GrandpaJustification; pub use light_import::light_block_import; -pub use observer::run_grandpa_observer; pub use voting_rule::{ BeforeBestBlockBy, ThreeQuartersOfTheUnfinalizedChain, VotingRule, VotingRulesBuilder }; @@ -516,7 +515,7 @@ fn register_finality_tracker_inherent_data_provider( } /// Parameters used to run Grandpa. -pub struct GrandpaParams { +pub struct GrandpaParams { /// Configuration for the GRANDPA service. pub config: Config, /// A link to the block import worker. @@ -531,14 +530,12 @@ pub struct GrandpaParams { pub telemetry_on_connect: Option>, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, - /// How to spawn background tasks. - pub executor: Sp, } /// Run a GRANDPA voter as a task. Provide configuration and a link to a /// block import worker that has already been instantiated with `block_import`. -pub fn run_grandpa_voter( - grandpa_params: GrandpaParams, +pub fn run_grandpa_voter( + grandpa_params: GrandpaParams, ) -> sp_blockchain::Result + Unpin + Send + 'static> where Block::Hash: Ord, B: Backend + 'static, @@ -551,19 +548,23 @@ pub fn run_grandpa_voter( RA: Send + Sync + 'static, X: futures::Future + Clone + Send + Unpin + 'static, Client: AuxStore, - Sp: futures::task::Spawn + 'static, { let GrandpaParams { - config, + mut config, link, network, inherent_data_providers, on_exit, telemetry_on_connect, voting_rule, - executor, } = grandpa_params; + // NOTE: we have recently removed `run_grandpa_observer` from the public + // API, I felt it is easier to just ignore this field rather than removing + // it from the config temporarily. This should be removed after #5013 is + // fixed and we re-add the observer to the public API. + config.observer_enabled = false; + let LinkHalf { client, select_chain, @@ -575,7 +576,6 @@ pub fn run_grandpa_voter( network, config.clone(), persistent_data.set_state.clone(), - &executor, ); register_finality_tracker_inherent_data_provider(client.clone(), &inherent_data_providers)?; @@ -862,26 +862,6 @@ where } } -#[deprecated(since = "1.1.0", note = "Please switch to run_grandpa_voter.")] -pub fn run_grandpa( - grandpa_params: GrandpaParams, -) -> sp_blockchain::Result + Send + 'static> where - Block::Hash: Ord, - B: Backend + 'static, - E: CallExecutor + Send + Sync + 'static, - N: NetworkT + Send + Sync + Clone + 'static, - SC: SelectChain + 'static, - NumberFor: BlockNumberOps, - DigestFor: Encode, - RA: Send + Sync + 'static, - VR: VotingRule> + Clone + 'static, - X: futures::Future + Clone + Send + Unpin + 'static, - Client: AuxStore, - Sp: futures::task::Spawn + 'static, -{ - run_grandpa_voter(grandpa_params) -} - /// When GRANDPA is not initialized we still need to register the finality /// tracker inherent provider which might be expected by the runtime for block /// authoring. Additionally, we register a gossip message validator that @@ -906,7 +886,10 @@ pub fn setup_disabled_grandpa( // We register the GRANDPA protocol so that we don't consider it an anomaly // to receive GRANDPA messages on the network. We don't process the // messages. - network.register_notifications_protocol(communication::GRANDPA_ENGINE_ID); + network.register_notifications_protocol( + communication::GRANDPA_ENGINE_ID, + From::from(communication::GRANDPA_PROTOCOL_NAME), + ); Ok(()) } diff --git a/client/finality-grandpa/src/light_import.rs b/client/finality-grandpa/src/light_import.rs index 4ae35167be8ba00c3f3e4dd92ecd84eb947de422..0181a93f1088cb35c46715a9db4f32b937950f21 100644 --- a/client/finality-grandpa/src/light_import.rs +++ b/client/finality-grandpa/src/light_import.rs @@ -257,7 +257,7 @@ fn do_import_block( DigestFor: Encode, J: ProvableJustification, { - let hash = block.post_header().hash(); + let hash = block.post_hash(); let number = block.header.number().clone(); // we don't want to finalize on `inner.import_block` @@ -682,26 +682,19 @@ pub mod tests { authority_set: LightAuthoritySet::genesis(vec![(AuthorityId::from_slice(&[1; 32]), 1)]), consensus_changes: ConsensusChanges::empty(), }; - let block = BlockImportParams { - origin: BlockOrigin::Own, - header: Header { + let mut block = BlockImportParams::new( + BlockOrigin::Own, + Header { number: 1, parent_hash: client.chain_info().best_hash, state_root: Default::default(), digest: Default::default(), extrinsics_root: Default::default(), }, - justification, - post_digests: Vec::new(), - body: None, - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: true, - import_existing: false, - }; + ); + block.justification = justification; + block.fork_choice = Some(ForkChoiceStrategy::LongestChain); + do_import_block::<_, _, _, TestJustification>( &client, &mut import_data, diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 7e6d1e7793a81bf008a29b8a757e4df58ddbb16a..8345a3420998ec3e2fa1dc170323c8968c044e50 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -150,12 +150,14 @@ fn grandpa_observer( /// listening for and validating GRANDPA commits instead of following the full /// protocol. Provide configuration and a link to a block import worker that has /// already been instantiated with `block_import`. -pub fn run_grandpa_observer( +/// NOTE: this is currently not part of the crate's public API since we don't consider +/// it stable enough to use on a live network. +#[allow(unused)] +pub fn run_grandpa_observer( config: Config, link: LinkHalf, network: N, on_exit: impl futures::Future + Clone + Send + Unpin + 'static, - executor: Sp, ) -> sp_blockchain::Result + Unpin + Send + 'static> where B: Backend + 'static, E: CallExecutor + Send + Sync + 'static, @@ -163,7 +165,6 @@ pub fn run_grandpa_observer( SC: SelectChain + 'static, NumberFor: BlockNumberOps, RA: Send + Sync + 'static, - Sp: futures::task::Spawn + 'static, Client: AuxStore, { let LinkHalf { @@ -177,7 +178,6 @@ pub fn run_grandpa_observer( network, config.clone(), persistent_data.set_state.clone(), - &executor, ); let observer_work = ObserverWork::new( @@ -380,7 +380,7 @@ mod tests { use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt}; use sc_network::PeerId; - use futures::executor::{self, ThreadPool}; + use futures::executor; /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. Regression /// test for bug introduced in d4fbb897c and fixed in b7af8b339. @@ -392,10 +392,8 @@ mod tests { /// network. #[test] fn observer_work_polls_underlying_network_bridge() { - let thread_pool = ThreadPool::new().unwrap(); - // Create a test network. - let (tester_fut, _network) = make_test_network(&thread_pool); + let (tester_fut, _network) = make_test_network(); let mut tester = executor::block_on(tester_fut); // Create an observer. diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 0f4d9dadd023f8b43ce8681e72784e254b2003ce..16b50ccb9f2265f9940154cb9af6ee89b7ba6768 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -19,7 +19,7 @@ use super::*; use environment::HasVoted; use sc_network_test::{ - Block, DummySpecialization, Hash, TestNetFactory, BlockImportAdapter, Peer, + Block, Hash, TestNetFactory, BlockImportAdapter, Peer, PeersClient, PassThroughVerifier, }; use sc_network::config::{ProtocolConfig, Roles, BoxFinalityProofRequestBuilder}; @@ -68,7 +68,7 @@ type PeerData = > > >; -type GrandpaPeer = Peer; +type GrandpaPeer = Peer; struct GrandpaTestNet { peers: Vec, @@ -90,7 +90,6 @@ impl GrandpaTestNet { } impl TestNetFactory for GrandpaTestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = PeerData; @@ -147,7 +146,7 @@ impl TestNetFactory for GrandpaTestNet { use crate::light_import::tests::light_block_import_without_justifications; let authorities_provider = Arc::new(self.test_config.clone()); - // forbid direct finalization using justification that cames with the block + // forbid direct finalization using justification that came with the block // => light clients will try to fetch finality proofs let import = light_block_import_without_justifications( client.clone(), @@ -171,7 +170,7 @@ impl TestNetFactory for GrandpaTestNet { fn make_finality_proof_provider( &self, client: PeersClient - ) -> Option>> { + ) -> Option>> { match client { PeersClient::Full(_, ref backend) => { let authorities_provider = Arc::new(self.test_config.clone()); @@ -387,7 +386,6 @@ fn block_until_complete(future: impl Future + Unpin, net: &Arc( runtime: &mut current_thread::Runtime, - threads_pool: &futures::executor::ThreadPool, blocks: u64, net: Arc>, peers: &[Ed25519Keyring], @@ -454,7 +452,6 @@ fn run_to_completion_with( on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); @@ -473,12 +470,11 @@ fn run_to_completion_with( fn run_to_completion( runtime: &mut current_thread::Runtime, - threads_pool: &futures::executor::ThreadPool, blocks: u64, net: Arc>, peers: &[Ed25519Keyring] ) -> u64 { - run_to_completion_with(runtime, threads_pool, blocks, net, peers, |_| None) + run_to_completion_with(runtime, blocks, net, peers, |_| None) } fn add_scheduled_change(block: &mut Block, change: ScheduledChange) { @@ -499,17 +495,10 @@ fn add_forced_change( )); } -fn thread_pool() -> futures::executor::ThreadPool { - futures::executor::ThreadPool::builder().pool_size(2) - .create() - .expect("never fails") -} - #[test] fn finalize_3_voters_no_observers() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -523,7 +512,7 @@ fn finalize_3_voters_no_observers() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 20, net.clone(), peers); + run_to_completion(&mut runtime, 20, net.clone(), peers); // normally there's no justification for finalized blocks assert!( @@ -535,7 +524,6 @@ fn finalize_3_voters_no_observers() { #[test] fn finalize_3_voters_1_full_observer() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -595,7 +583,6 @@ fn finalize_3_voters_1_full_observer() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; voters.push(run_grandpa_voter(grandpa_params).expect("all in order with client and network")); @@ -641,7 +628,6 @@ fn transition_3_voters_twice_1_full_observer() { let net = Arc::new(Mutex::new(GrandpaTestNet::new(api, 8))); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); net.lock().peer(0).push_blocks(1, false); net.lock().block_until_sync(&mut runtime); @@ -760,7 +746,6 @@ fn transition_3_voters_twice_1_full_observer() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); @@ -776,7 +761,6 @@ fn transition_3_voters_twice_1_full_observer() { #[test] fn justification_is_emitted_when_consensus_data_changes() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 3); @@ -785,7 +769,7 @@ fn justification_is_emitted_when_consensus_data_changes() { net.peer(0).push_authorities_change_block(new_authorities); net.block_until_sync(&mut runtime); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 1, net.clone(), peers); + run_to_completion(&mut runtime, 1, net.clone(), peers); // ... and check that there's justification for block#1 assert!(net.lock().peer(0).client().justification(&BlockId::Number(1)).unwrap().is_some(), @@ -795,7 +779,6 @@ fn justification_is_emitted_when_consensus_data_changes() { #[test] fn justification_is_generated_periodically() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -804,7 +787,7 @@ fn justification_is_generated_periodically() { net.block_until_sync(&mut runtime); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 32, net.clone(), peers); + run_to_completion(&mut runtime, 32, net.clone(), peers); // when block#32 (justification_period) is finalized, justification // is required => generated @@ -835,7 +818,6 @@ fn consensus_changes_works() { #[test] fn sync_justifications_on_change_blocks() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers_b); @@ -867,7 +849,7 @@ fn sync_justifications_on_change_blocks() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 25, net.clone(), peers_a); + run_to_completion(&mut runtime, 25, net.clone(), peers_a); // the first 3 peers are grandpa voters and therefore have already finalized // block 21 and stored a justification @@ -890,7 +872,6 @@ fn sync_justifications_on_change_blocks() { fn finalizes_multiple_pending_changes_in_order() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Dave, Ed25519Keyring::Eve, Ed25519Keyring::Ferdie]; @@ -944,14 +925,13 @@ fn finalizes_multiple_pending_changes_in_order() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 30, net.clone(), all_peers); + run_to_completion(&mut runtime, 30, net.clone(), all_peers); } #[test] fn force_change_to_new_set() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // two of these guys are offline. let genesis_authorities = &[ Ed25519Keyring::Alice, @@ -1001,8 +981,8 @@ fn force_change_to_new_set() { // it will only finalize if the forced transition happens. // we add_blocks after the voters are spawned because otherwise - // the link-halfs have the wrong AuthoritySet - run_to_completion(&mut runtime, &threads_pool, 25, net, peers_a); + // the link-halves have the wrong AuthoritySet + run_to_completion(&mut runtime, 25, net, peers_a); } #[test] @@ -1030,20 +1010,11 @@ fn allows_reimporting_change_blocks() { let block = || { let block = block.clone(); - BlockImportParams { - origin: BlockOrigin::File, - header: block.header, - justification: None, - post_digests: Vec::new(), - body: Some(block.extrinsics), - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - } + let mut import = BlockImportParams::new(BlockOrigin::File, block.header); + import.body = Some(block.extrinsics); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + + import }; assert_eq!( @@ -1090,20 +1061,12 @@ fn test_bad_justification() { let block = || { let block = block.clone(); - BlockImportParams { - origin: BlockOrigin::File, - header: block.header, - justification: Some(Vec::new()), - post_digests: Vec::new(), - body: Some(block.extrinsics), - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - } + let mut import = BlockImportParams::new(BlockOrigin::File, block.header); + import.justification = Some(Vec::new()); + import.body = Some(block.extrinsics); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + + import }; assert_eq!( @@ -1132,7 +1095,6 @@ fn voter_persists_its_votes() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // we have two authorities but we'll only be running the voter for alice // we are going to be listening for the prevotes it casts @@ -1171,7 +1133,6 @@ fn voter_persists_its_votes() { net: Arc>, client: PeersClient, keystore: KeyStorePtr, - threads_pool: futures::executor::ThreadPool, } impl Future for ResettableVoter { @@ -1210,7 +1171,6 @@ fn voter_persists_its_votes() { on_exit: Exit, telemetry_on_connect: None, voting_rule: VotingRulesBuilder::default().build(), - executor: this.threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params) @@ -1242,7 +1202,6 @@ fn voter_persists_its_votes() { net: net.clone(), client: client.clone(), keystore, - threads_pool: threads_pool.clone(), }.unit_error().compat()); } @@ -1278,7 +1237,6 @@ fn voter_persists_its_votes() { net.lock().peers[1].network_service().clone(), config.clone(), set_state, - &threads_pool, ); let (round_rx, round_tx) = network.round_communication( @@ -1289,6 +1247,11 @@ fn voter_persists_its_votes() { HasVoted::No, ); + runtime.spawn( + network.map_err(|e| panic!("network bridge should not error: {:?}", e)) + .compat(), + ); + let round_tx = Arc::new(Mutex::new(round_tx)); let exit_tx = Arc::new(Mutex::new(Some(exit_tx))); @@ -1389,7 +1352,6 @@ fn voter_persists_its_votes() { fn finalize_3_voters_1_light_observer() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let authorities = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(authorities); @@ -1406,12 +1368,14 @@ fn finalize_3_voters_1_light_observer() { let link = net.lock().peer(3).data.lock().take().expect("link initialized on startup; qed"); let finality_notifications = net.lock().peer(3).client().finality_notification_stream() - .take_while(|n| future::ready(n.header.number() < &20)) + .take_while(|n| { + future::ready(n.header.number() < &20) + }) .collect::>(); - run_to_completion_with(&mut runtime, &threads_pool, 20, net.clone(), authorities, |executor| { + run_to_completion_with(&mut runtime, 20, net.clone(), authorities, |executor| { executor.spawn( - run_grandpa_observer( + observer::run_grandpa_observer( Config { gossip_duration: TEST_GOSSIP_DURATION, justification_period: 32, @@ -1423,7 +1387,6 @@ fn finalize_3_voters_1_light_observer() { link, net.lock().peers[3].network_service().clone(), Exit, - threads_pool.clone(), ).unwrap().unit_error().compat() ).unwrap(); @@ -1435,7 +1398,6 @@ fn finalize_3_voters_1_light_observer() { fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { let _ = ::env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 1); @@ -1445,7 +1407,7 @@ fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { // && instead fetches finality proof for block #1 net.peer(0).push_authorities_change_block(vec![sp_consensus_babe::AuthorityId::from_slice(&[42; 32])]); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 1, net.clone(), peers); + run_to_completion(&mut runtime, 1, net.clone(), peers); net.lock().block_until_sync(&mut runtime); // check that the block#1 is finalized on light client @@ -1467,7 +1429,6 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ let _ = ::env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // two of these guys are offline. let genesis_authorities = if FORCE_CHANGE { @@ -1515,7 +1476,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ net.lock().block_until_sync(&mut runtime); // finalize block #11 on full clients - run_to_completion(&mut runtime, &threads_pool, 11, net.clone(), peers_a); + run_to_completion(&mut runtime, 11, net.clone(), peers_a); // request finalization by light client net.lock().add_light_peer(&GrandpaTestNet::default_config()); @@ -1532,7 +1493,6 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ fn voter_catches_up_to_latest_round_when_behind() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers); @@ -1560,7 +1520,6 @@ fn voter_catches_up_to_latest_round_when_behind() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) @@ -1652,8 +1611,6 @@ fn grandpa_environment_respects_voting_rules() { use finality_grandpa::Chain; use sc_network_test::TestClient; - let threads_pool = thread_pool(); - let peers = &[Ed25519Keyring::Alice]; let voters = make_ids(peers); @@ -1684,7 +1641,6 @@ fn grandpa_environment_respects_voting_rules() { network_service.clone(), config.clone(), set_state.clone(), - &threads_pool, ); Environment { @@ -1831,23 +1787,13 @@ fn imports_justification_for_regular_blocks_on_import() { }; // we import the block with justification attached - let block = BlockImportParams { - origin: BlockOrigin::File, - header: block.header, - justification: Some(justification.encode()), - post_digests: Vec::new(), - body: Some(block.extrinsics), - storage_changes: None, - finalized: false, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; + let mut import = BlockImportParams::new(BlockOrigin::File, block.header); + import.justification = Some(justification.encode()); + import.body = Some(block.extrinsics); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); assert_eq!( - block_import.import_block(block, HashMap::new()).unwrap(), + block_import.import_block(import, HashMap::new()).unwrap(), ImportResult::Imported(ImportedAux { needs_justification: false, clear_justification_requests: false, diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..c7a6c0843254001e1e9e32b506b9a75d7465745f --- /dev/null +++ b/client/informant/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sc-informant" +version = "0.8.0-alpha.1" +authors = ["Parity Technologies "] +description = "Substrate informant." +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[dependencies] +ansi_term = "0.12.1" +futures = "0.3.1" +log = "0.4.8" +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } +wasm-timer = "0.2" +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sc-service = { version = "0.8.0-alpha.1", default-features = false, path = "../service" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } diff --git a/client/cli/src/informant/display.rs b/client/informant/src/display.rs similarity index 76% rename from client/cli/src/informant/display.rs rename to client/informant/src/display.rs index 199635e7c78833e46c5f076b1d19a060dbc528d4..53c9697868348cb9a73328d26a459601196cd240 100644 --- a/client/cli/src/informant/display.rs +++ b/client/informant/src/display.rs @@ -20,7 +20,9 @@ use log::info; use sc_network::SyncState; use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating}; use sc_service::NetworkStatus; -use std::{convert::{TryFrom, TryInto}, fmt, time}; +use std::{convert::{TryFrom, TryInto}, fmt}; +use wasm_timer::Instant; +use crate::OutputFormat; /// State of the informant display system. /// @@ -40,15 +42,18 @@ pub struct InformantDisplay { /// `None` if `display` has never been called. last_number: Option>, /// The last time `display` or `new` has been called. - last_update: time::Instant, + last_update: Instant, + /// The format to print output in. + format: OutputFormat, } impl InformantDisplay { /// Builds a new informant display system. - pub fn new() -> InformantDisplay { + pub fn new(format: OutputFormat) -> InformantDisplay { InformantDisplay { last_number: None, - last_update: time::Instant::now(), + last_update: Instant::now(), + format, } } @@ -56,8 +61,10 @@ impl InformantDisplay { pub fn display(&mut self, info: &ClientInfo, net_status: NetworkStatus) { let best_number = info.chain.best_number; let best_hash = info.chain.best_hash; + let finalized_number = info.chain.finalized_number; + let num_connected_peers = net_status.num_connected_peers; let speed = speed::(best_number, self.last_number, self.last_update); - self.last_update = time::Instant::now(); + self.last_update = Instant::now(); self.last_number = Some(best_number); let (status, target) = match (net_status.sync_state, net_status.best_seen_block) { @@ -66,19 +73,35 @@ impl InformantDisplay { (SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed), format!(", target=#{}", n)), }; - info!( - target: "substrate", - "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", - Colour::White.bold().paint(&status), - target, - Colour::White.bold().paint(format!("{}", net_status.num_connected_peers)), - Colour::White.paint(format!("{}", best_number)), - best_hash, - Colour::White.paint(format!("{}", info.chain.finalized_number)), - info.chain.finalized_hash, - TransferRateFormat(net_status.average_download_per_sec), - TransferRateFormat(net_status.average_upload_per_sec), - ); + if self.format == OutputFormat::Coloured { + info!( + target: "substrate", + "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + Colour::White.bold().paint(&status), + target, + Colour::White.bold().paint(format!("{}", num_connected_peers)), + Colour::White.paint(format!("{}", best_number)), + best_hash, + Colour::White.paint(format!("{}", finalized_number)), + info.chain.finalized_hash, + TransferRateFormat(net_status.average_download_per_sec), + TransferRateFormat(net_status.average_upload_per_sec), + ); + } else { + info!( + target: "substrate", + "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + status, + target, + num_connected_peers, + best_number, + best_hash, + finalized_number, + info.chain.finalized_hash, + TransferRateFormat(net_status.average_download_per_sec), + TransferRateFormat(net_status.average_upload_per_sec), + ); + } } } @@ -87,7 +110,7 @@ impl InformantDisplay { fn speed( best_number: NumberFor, last_number: Option>, - last_update: time::Instant + last_update: Instant ) -> String { // Number of milliseconds elapsed since last time. let elapsed_ms = { diff --git a/client/cli/src/informant.rs b/client/informant/src/lib.rs similarity index 90% rename from client/cli/src/informant.rs rename to client/informant/src/lib.rs index 9e7c5044e046d0f2d719d45f2fd3ea64637ab5ce..699dcfdd7425e2b7b9e0f82eba481df2e81bf951 100644 --- a/client/cli/src/informant.rs +++ b/client/informant/src/lib.rs @@ -25,12 +25,19 @@ use std::time::Duration; mod display; +/// The format to print telemetry output in. +#[derive(PartialEq)] +pub enum OutputFormat { + Coloured, + Plain, +} + /// Creates an informant in the form of a `Future` that must be polled regularly. -pub fn build(service: &impl AbstractService) -> impl futures::Future { +pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futures::Future { let client = service.client(); let pool = service.transaction_pool(); - let mut display = display::InformantDisplay::new(); + let mut display = display::InformantDisplay::new(format); let display_notifications = service .network_status(Duration::from_millis(5000)) @@ -41,6 +48,7 @@ pub fn build(service: &impl AbstractService) -> impl futures::Future"] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0", path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../primitives/application-crypto" } hex = "0.4.0" rand = "0.7.2" serde_json = "1.0.41" diff --git a/client/keystore/src/lib.rs b/client/keystore/src/lib.rs index ef81c40c10f51f91a6aea3f17797e38ef29141ae..087ddf326de0141cc011cf4b046779193b4ba2a1 100644 --- a/client/keystore/src/lib.rs +++ b/client/keystore/src/lib.rs @@ -19,13 +19,10 @@ #![warn(missing_docs)] use std::{collections::HashMap, path::PathBuf, fs::{self, File}, io::{self, Write}, sync::Arc}; - use sp_core::{ crypto::{KeyTypeId, Pair as PairT, Public, IsWrappedBy, Protected}, traits::BareCryptoStore, }; - use sp_application_crypto::{AppKey, AppPublic, AppPair, ed25519, sr25519}; - use parking_lot::RwLock; /// Keystore pointer @@ -260,7 +257,7 @@ impl Store { /// Get public keys of all stored keys that match the key type. /// /// This will just use the type of the public key (a list of which to be returned) in order - /// to determine the key type. Unless you use a specialised application-type public key, then + /// to determine the key type. Unless you use a specialized application-type public key, then /// this only give you keys registered under generic cryptography, and will not return keys /// registered under the application type. pub fn public_keys(&self) -> Result> { diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 22f22b4856214615c68daa559c692d3a356e77bc..e42306c340eaae282bc0f9969c78a9210cc5d0dd 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -1,18 +1,20 @@ [package] description = "Gossiping for the Substrate network protocol" name = "sc-network-gossip" -version = "0.8.0" +version = "0.8.0-alpha.1" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] +futures = "0.3.1" +futures-timer = "3.0.1" +libp2p = { version = "0.16.1", default-features = false, features = ["libp2p-websocket"] } log = "0.4.8" -futures = { version = "0.3.1", features = ["compat"] } -futures01 = { package = "futures", version = "0.1.29" } -futures-timer = "0.4.0" -libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } -lru = "0.1.2" +lru = "0.4.3" parking_lot = "0.10.0" -sc-network = { version = "0.8", path = "../network" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +wasm-timer = "0.2" diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index d6d6805b3e56cc750a97e9c01eaa25767419fc9d..c911766aba40a4f41b0588ae2c33a9a51d5daab4 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -15,16 +15,16 @@ // along with Substrate. If not, see . use crate::{Network, Validator}; -use crate::state_machine::{ConsensusGossip, TopicNotification}; +use crate::state_machine::{ConsensusGossip, TopicNotification, PERIODIC_MAINTENANCE_INTERVAL}; use sc_network::message::generic::ConsensusMessage; use sc_network::{Event, ReputationChange}; -use futures::{prelude::*, channel::mpsc, compat::Compat01As03, task::SpawnExt as _}; +use futures::{prelude::*, channel::mpsc}; use libp2p::PeerId; use parking_lot::Mutex; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::{sync::Arc, time::Duration}; +use std::{borrow::Cow, pin::Pin, sync::Arc, task::{Context, Poll}}; /// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on /// top of it. @@ -36,28 +36,36 @@ pub struct GossipEngine { struct GossipEngineInner { state_machine: ConsensusGossip, network: Box + Send>, + periodic_maintenance_interval: futures_timer::Delay, + network_event_stream: Pin + Send>>, + engine_id: ConsensusEngineId, } +impl Unpin for GossipEngineInner {} + impl GossipEngine { /// Create a new instance. pub fn new + Send + Clone + 'static>( mut network: N, - executor: &impl futures::task::Spawn, engine_id: ConsensusEngineId, + protocol_name: impl Into>, validator: Arc>, ) -> Self where B: 'static { let mut state_machine = ConsensusGossip::new(); // We grab the event stream before registering the notifications protocol, otherwise we // might miss events. - let event_stream = network.event_stream(); + let network_event_stream = network.event_stream(); - network.register_notifications_protocol(engine_id); + network.register_notifications_protocol(engine_id, protocol_name.into()); state_machine.register_validator(&mut network, engine_id, validator); let inner = Arc::new(Mutex::new(GossipEngineInner { state_machine, network: Box::new(network), + periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), + network_event_stream, + engine_id, })); let gossip_engine = GossipEngine { @@ -65,72 +73,6 @@ impl GossipEngine { engine_id, }; - let res = executor.spawn({ - let inner = Arc::downgrade(&inner); - async move { - loop { - let _ = futures_timer::Delay::new(Duration::from_millis(1100)).await; - if let Some(inner) = inner.upgrade() { - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.tick(&mut *inner.network); - } else { - // We reach this branch if the `Arc` has no reference - // left. We can now let the task end. - break; - } - } - } - }); - - // Note: we consider the chances of an error to spawn a background task almost null. - if res.is_err() { - log::error!(target: "gossip", "Failed to spawn background task"); - } - - let res = executor.spawn(async move { - let mut stream = Compat01As03::new(event_stream); - while let Some(Ok(event)) = stream.next().await { - match event { - Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { - if msg_engine_id != engine_id { - continue; - } - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.new_peer(&mut *inner.network, remote, roles); - } - Event::NotificationStreamClosed { remote, engine_id: msg_engine_id } => { - if msg_engine_id != engine_id { - continue; - } - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.peer_disconnected(&mut *inner.network, remote); - }, - Event::NotificationsReceived { remote, messages } => { - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.on_incoming( - &mut *inner.network, - remote, - messages.into_iter() - .filter_map(|(engine, data)| if engine == engine_id { - Some(ConsensusMessage { engine_id: engine, data: data.to_vec() }) - } else { None }) - .collect() - ); - }, - Event::Dht(_) => {} - } - } - }); - - // Note: we consider the chances of an error to spawn a background task almost null. - if res.is_err() { - log::error!(target: "gossip", "Failed to spawn background task"); - } - gossip_engine } @@ -222,6 +164,59 @@ impl GossipEngine { } } +impl Future for GossipEngine { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + self.inner.lock().poll_unpin(cx) + } +} + +impl Future for GossipEngineInner { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = &mut *self; + + while let Poll::Ready(Some(event)) = this.network_event_stream.poll_next_unpin(cx) { + match event { + Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { + if msg_engine_id != this.engine_id { + continue; + } + this.state_machine.new_peer(&mut *this.network, remote, roles); + } + Event::NotificationStreamClosed { remote, engine_id: msg_engine_id } => { + if msg_engine_id != this.engine_id { + continue; + } + this.state_machine.peer_disconnected(&mut *this.network, remote); + }, + Event::NotificationsReceived { remote, messages } => { + let engine_id = this.engine_id.clone(); + this.state_machine.on_incoming( + &mut *this.network, + remote, + messages.into_iter() + .filter_map(|(engine, data)| if engine == engine_id { + Some(ConsensusMessage { engine_id: engine, data: data.to_vec() }) + } else { None }) + .collect() + ); + }, + Event::Dht(_) => {} + } + } + + while let Poll::Ready(()) = this.periodic_maintenance_interval.poll_unpin(cx) { + this.periodic_maintenance_interval.reset(PERIODIC_MAINTENANCE_INTERVAL); + this.state_machine.tick(&mut *this.network); + } + + Poll::Pending + } +} + impl Clone for GossipEngine { fn clone(&self) -> Self { GossipEngine { diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs index 705a27210ac53de386aedd4c9d681467d9acda55..4e4d32366f29d4c907561fc844e107edc60b8627 100644 --- a/client/network-gossip/src/lib.rs +++ b/client/network-gossip/src/lib.rs @@ -59,9 +59,9 @@ pub use self::state_machine::TopicNotification; pub use self::validator::{DiscardAll, MessageIntent, Validator, ValidatorContext, ValidationResult}; use futures::prelude::*; -use sc_network::{specialization::NetworkSpecialization, Event, ExHashT, NetworkService, PeerId, ReputationChange}; +use sc_network::{Event, ExHashT, NetworkService, PeerId, ReputationChange}; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::sync::Arc; +use std::{borrow::Cow, pin::Pin, sync::Arc}; mod bridge; mod state_machine; @@ -70,7 +70,7 @@ mod validator; /// Abstraction over a network. pub trait Network { /// Returns a stream of events representing what happens on the network. - fn event_stream(&self) -> Box + Send>; + fn event_stream(&self) -> Pin + Send>>; /// Adjust the reputation of a node. fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange); @@ -86,7 +86,8 @@ pub trait Network { /// See the documentation of [`NetworkService:register_notifications_protocol`] for more information. fn register_notifications_protocol( &self, - engine_id: ConsensusEngineId + engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, ); /// Notify everyone we're connected to that we have the given block. @@ -96,9 +97,9 @@ pub trait Network { fn announce(&self, block: B::Hash, associated_data: Vec); } -impl, H: ExHashT> Network for Arc> { - fn event_stream(&self) -> Box + Send> { - Box::new(NetworkService::event_stream(self).map(|v| Ok::<_, ()>(v)).compat()) +impl Network for Arc> { + fn event_stream(&self) -> Pin + Send>> { + Box::pin(NetworkService::event_stream(self)) } fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange) { @@ -116,8 +117,9 @@ impl, H: ExHashT> Network for Arc, ) { - NetworkService::register_notifications_protocol(self, engine_id) + NetworkService::register_notifications_protocol(self, engine_id, protocol_name) } fn announce(&self, block: B::Hash, associated_data: Vec) { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index d1931b1bd29d81411a53a83b8d2de6a976f19d20..26433e63ec3ea7e09a6185628f4a5a3ec894e484 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -28,12 +28,15 @@ use sp_runtime::traits::{Block as BlockT, Hash, HashFor}; use sp_runtime::ConsensusEngineId; pub use sc_network::message::generic::{Message, ConsensusMessage}; use sc_network::config::Roles; +use wasm_timer::Instant; // FIXME: Add additional spam/DoS attack protection: https://github.com/paritytech/substrate/issues/1115 const KNOWN_MESSAGES_CACHE_SIZE: usize = 4096; const REBROADCAST_INTERVAL: time::Duration = time::Duration::from_secs(30); +pub(crate) const PERIODIC_MAINTENANCE_INTERVAL: time::Duration = time::Duration::from_millis(1100); + mod rep { use sc_network::ReputationChange as Rep; /// Reputation change when a peer sends us a gossip message that we didn't know about. @@ -42,7 +45,7 @@ mod rep { pub const DUPLICATE_GOSSIP: Rep = Rep::new(-(1 << 2), "Duplicate gossip"); /// Reputation change when a peer sends us a gossip message for an unknown engine, whatever that /// means. - pub const UNKNOWN_GOSSIP: Rep = Rep::new(-(1 << 6), "Unknown gossup message engine id"); + pub const UNKNOWN_GOSSIP: Rep = Rep::new(-(1 << 6), "Unknown gossip message engine id"); /// Reputation change when a peer sends a message from a topic it isn't registered on. pub const UNREGISTERED_TOPIC: Rep = Rep::new(-(1 << 10), "Unregistered gossip message topic"); } @@ -165,7 +168,7 @@ pub struct ConsensusGossip { messages: Vec>, known_messages: LruCache, validators: HashMap>>, - next_broadcast: time::Instant, + next_broadcast: Instant, } impl ConsensusGossip { @@ -177,7 +180,7 @@ impl ConsensusGossip { messages: Default::default(), known_messages: LruCache::new(KNOWN_MESSAGES_CACHE_SIZE), validators: Default::default(), - next_broadcast: time::Instant::now() + REBROADCAST_INTERVAL, + next_broadcast: Instant::now() + REBROADCAST_INTERVAL, } } @@ -260,9 +263,9 @@ impl ConsensusGossip { /// Perform periodic maintenance pub fn tick(&mut self, network: &mut dyn Network) { self.collect_garbage(); - if time::Instant::now() >= self.next_broadcast { + if Instant::now() >= self.next_broadcast { self.rebroadcast(network); - self.next_broadcast = time::Instant::now() + REBROADCAST_INTERVAL; + self.next_broadcast = Instant::now() + REBROADCAST_INTERVAL; } } diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index e506a61c750fddae4bb4d798577b16aef51e0d87..2d50146653a08201d8c3efde1b1e5ea49267d7bb 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -1,10 +1,15 @@ [package] description = "Substrate network protocol" name = "sc-network" -version = "0.8.0" +version = "0.8.0-alpha.1" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[build-dependencies] +prost-build = "0.6.1" [dependencies] bitflags = "1.2.0" @@ -14,48 +19,54 @@ derive_more = "0.99.2" either = "1.5.3" erased-serde = "0.3.9" fnv = "1.0.6" -fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-alpha.1", path = "../../utils/fork-tree" } futures = "0.3.1" futures_codec = "0.3.3" -futures-timer = "0.4.0" -libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } +futures-timer = "3.0.1" +wasm-timer = "0.2" +libp2p = { version = "0.16.1", default-features = false, features = ["libp2p-websocket"] } linked-hash-map = "0.5.2" linked_hash_set = "0.1.3" log = "0.4.8" lru = "0.4.0" +nohash-hasher = "0.2.0" parking_lot = "0.10.0" +prost = "0.6.1" rand = "0.7.2" rustc-hex = "2.0.1" -sc-block-builder = { version = "0.8", path = "../block-builder" } -sc-client = { version = "0.8", path = "../" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-peerset = { version = "2.0.0", path = "../peerset" } +sc-block-builder = { version = "0.8.0-alpha.1", path = "../block-builder" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-peerset = { version = "2.0.0-alpha.1", path = "../peerset" } +pin-project = "0.4.6" serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } slog_derive = "0.2.0" smallvec = "0.6.10" -sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-keyring = { version = "2.0.0", optional = true, path = "../../primitives/keyring" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -substrate-test-client = { version = "2.0.0", optional = true, path = "../../test-utils/client" } -substrate-test-runtime-client = { version = "2.0.0", optional = true, path = "../../test-utils/runtime/client" } -unsigned-varint = { version = "0.3.0", features = ["codec"] } +sp-arithmetic = { version = "2.0.0-alpha.1", path = "../../primitives/arithmetic" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +thiserror = "1" +unsigned-varint = { version = "0.3.1", features = ["futures", "futures-codec"] } void = "1.0.2" zeroize = "1.0.0" [dev-dependencies] +async-std = "1.5" +assert_matches = "1.3" env_logger = "0.7.0" quickcheck = "0.9.0" rand = "0.7.2" -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } +sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tempfile = "3.1.0" [features] default = [] -test-helpers = ["sp-keyring", "substrate-test-runtime-client"] + diff --git a/client/network/build.rs b/client/network/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..0fd1f128660e9aaec36c4ac14e405f77e9588096 --- /dev/null +++ b/client/network/build.rs @@ -0,0 +1,8 @@ +const PROTOS: &[&str] = &[ + "src/protocol/schema/api.v1.proto", + "src/protocol/schema/light.v1.proto" +]; + +fn main() { + prost_build::compile_protos(PROTOS, &["src/protocol"]).unwrap(); +} diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 8b903cec351f5fcbca4e5601cf70b91b936469b4..a03a6caa2f5ee9d3c9183da22ae99732683e81b9 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -16,15 +16,13 @@ use crate::{ debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, DiscoveryNetBehaviour, - Event, protocol::event::DhtEvent + Event, protocol::event::DhtEvent, ExHashT, }; -use crate::{ExHashT, specialization::NetworkSpecialization}; -use crate::protocol::{CustomMessageOutcome, Protocol}; +use crate::protocol::{self, light_client_handler, CustomMessageOutcome, Protocol}; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; use libp2p::kad::record; -use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess}; -use libp2p::core::{nodes::Substream, muxing::StreamMuxerBox}; +use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters}; use log::debug; use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}}; use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justification}; @@ -34,15 +32,18 @@ use void; /// General behaviour of the network. Combines all protocols together. #[derive(NetworkBehaviour)] #[behaviour(out_event = "BehaviourOut", poll_method = "poll")] -pub struct Behaviour, H: ExHashT> { +pub struct Behaviour { /// All the substrate-specific protocols. - substrate: Protocol, + substrate: Protocol, /// Periodically pings and identifies the nodes we are connected to, and store information in a /// cache. - debug_info: debug_info::DebugInfoBehaviour>, + debug_info: debug_info::DebugInfoBehaviour, /// Discovers nodes of the network. - discovery: DiscoveryBehaviour>, - + discovery: DiscoveryBehaviour, + /// Block request handling. + block_requests: protocol::BlockRequests, + /// Light client request handling. + light_client_handler: protocol::LightClientHandler, /// Queue of events to produce for the outside. #[behaviour(ignore)] events: Vec>, @@ -56,15 +57,18 @@ pub enum BehaviourOut { Event(Event), } -impl, H: ExHashT> Behaviour { +impl Behaviour { /// Builds a new `Behaviour`. pub async fn new( - substrate: Protocol, + substrate: Protocol, user_agent: String, local_public_key: PublicKey, known_addresses: Vec<(PeerId, Multiaddr)>, enable_mdns: bool, allow_private_ipv4: bool, + discovery_only_if_under_num: u64, + block_requests: protocol::BlockRequests, + light_client_handler: protocol::LightClientHandler, ) -> Self { Behaviour { substrate, @@ -73,9 +77,12 @@ impl, H: ExHashT> Behaviour { local_public_key, known_addresses, enable_mdns, - allow_private_ipv4 + allow_private_ipv4, + discovery_only_if_under_num, ).await, - events: Vec::new(), + block_requests, + light_client_handler, + events: Vec::new() } } @@ -99,12 +106,12 @@ impl, H: ExHashT> Behaviour { } /// Returns a shared reference to the user protocol. - pub fn user_protocol(&self) -> &Protocol { + pub fn user_protocol(&self) -> &Protocol { &self.substrate } /// Returns a mutable reference to the user protocol. - pub fn user_protocol_mut(&mut self) -> &mut Protocol { + pub fn user_protocol_mut(&mut self) -> &mut Protocol { &mut self.substrate } @@ -117,17 +124,23 @@ impl, H: ExHashT> Behaviour { pub fn put_value(&mut self, key: record::Key, value: Vec) { self.discovery.put_value(key, value); } + + /// Issue a light client request. + #[allow(unused)] + pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { + self.light_client_handler.request(r) + } } -impl, H: ExHashT> NetworkBehaviourEventProcess for -Behaviour { +impl NetworkBehaviourEventProcess for +Behaviour { fn inject_event(&mut self, event: void::Void) { void::unreachable(event) } } -impl, H: ExHashT> NetworkBehaviourEventProcess> for -Behaviour { +impl NetworkBehaviourEventProcess> for +Behaviour { fn inject_event(&mut self, event: CustomMessageOutcome) { match event { CustomMessageOutcome::BlockImport(origin, blocks) => @@ -160,8 +173,8 @@ Behaviour { } } -impl, H: ExHashT> NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, event: debug_info::DebugInfoEvent) { let debug_info::DebugInfoEvent::Identified { peer_id, mut info } = event; if info.listen_addrs.len() > 30 { @@ -178,8 +191,8 @@ impl, H: ExHashT> NetworkBehaviourEventPr } } -impl, H: ExHashT> NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, out: DiscoveryOut) { match out { DiscoveryOut::UnroutablePeer(_peer_id) => { @@ -207,8 +220,8 @@ impl, H: ExHashT> NetworkBehaviourEventPr } } -impl, H: ExHashT> Behaviour { - fn poll(&mut self, _: &mut Context) -> Poll>> { +impl Behaviour { + fn poll(&mut self, _: &mut Context, _: &mut impl PollParameters) -> Poll>> { if !self.events.is_empty() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))) } diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 6cf2587fe47cb4ad5a344cea4bfc404b49b2e2d7..f5cad5977fc470fb43f2681aab54ab5fb0d968ce 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -19,12 +19,18 @@ //! The [`Params`] struct is the struct that must be passed in order to initialize the networking. //! See the documentation of [`Params`]. -pub use crate::protocol::ProtocolConfig; +pub use crate::chain::{Client, FinalityProofProvider}; +pub use crate::on_demand_layer::OnDemand; +pub use crate::service::TransactionPool; pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr}; -use crate::chain::{Client, FinalityProofProvider}; -use crate::on_demand_layer::OnDemand; -use crate::service::{ExHashT, TransactionPool}; +// Note: this re-export shouldn't be part of the public API of the crate and will be removed in +// the future. +#[doc(hidden)] +pub use crate::protocol::ProtocolConfig; + +use crate::service::ExHashT; + use bitflags::bitflags; use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue}; use sp_runtime::traits::{Block as BlockT}; @@ -37,7 +43,7 @@ use std::{error::Error, fs, io::{self, Write}, net::Ipv4Addr, path::{Path, PathB use zeroize::Zeroize; /// Network initialization parameters. -pub struct Params { +pub struct Params { /// Assigned roles for our node (full, light, ...). pub roles: Roles, @@ -82,9 +88,6 @@ pub struct Params { /// valid. pub import_queue: Box>, - /// Customization of the network. Use this to plug additional networking capabilities. - pub specialization: S, - /// Type to check incoming block announcements. pub block_announce_validator: Box + Send>, } @@ -120,6 +123,12 @@ impl Roles { } } +impl fmt::Display for Roles { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + impl codec::Encode for Roles { fn encode_to(&self, dest: &mut T) { dest.push_byte(self.bits()) @@ -241,9 +250,9 @@ impl From for ParseErr { #[derive(Clone, Debug)] pub struct NetworkConfiguration { /// Directory path to store general network configuration. None means nothing will be saved. - pub config_path: Option, + pub config_path: Option, /// Directory path to store network-specific configuration. None means nothing will be saved. - pub net_config_path: Option, + pub net_config_path: Option, /// Multiaddresses to listen for incoming connections. pub listen_addresses: Vec, /// Multiaddresses to advertise. Detected automatically if empty. @@ -292,6 +301,7 @@ impl Default for NetworkConfiguration { enable_mdns: false, allow_private_ipv4: true, wasm_external_transport: None, + use_yamux_flow_control: false, }, max_parallel_downloads: 5, } @@ -337,8 +347,9 @@ pub enum TransportConfig { enable_mdns: bool, /// If true, allow connecting to private IPv4 addresses (as defined in - /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address has been passed in - /// [`NetworkConfiguration::reserved_nodes`] or [`NetworkConfiguration::boot_nodes`]. + /// [RFC1918](https://tools.ietf.org/html/rfc1918)). Irrelevant for addresses that have + /// been passed in [`NetworkConfiguration::reserved_nodes`] or + /// [`NetworkConfiguration::boot_nodes`]. allow_private_ipv4: bool, /// Optional external implementation of a libp2p transport. Used in WASM contexts where we @@ -348,6 +359,8 @@ pub enum TransportConfig { /// This parameter exists whatever the target platform is, but it is expected to be set to /// `Some` only when compiling for WASM. wasm_external_transport: Option, + /// Use flow control for yamux streams if set to true. + use_yamux_flow_control: bool, }, /// Only allow connections within the same process. diff --git a/client/network/src/debug_info.rs b/client/network/src/debug_info.rs index b06e275d1d6ac3600128bae5ea89b0b3545abdd5..17fb622f7cd3cab44f3b17ab508d7f1a3b258882 100644 --- a/client/network/src/debug_info.rs +++ b/client/network/src/debug_info.rs @@ -28,7 +28,8 @@ use std::error; use std::collections::hash_map::Entry; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; use crate::utils::interval; /// Time after we disconnect from a node before we purge its information from the cache. @@ -38,11 +39,11 @@ const GARBAGE_COLLECT_INTERVAL: Duration = Duration::from_secs(2 * 60); /// Implementation of `NetworkBehaviour` that holds information about nodes in cache for diagnostic /// purposes. -pub struct DebugInfoBehaviour { +pub struct DebugInfoBehaviour { /// Periodically ping nodes, and close the connection if it's unresponsive. - ping: Ping, + ping: Ping, /// Periodically identifies the remote and responds to incoming requests. - identify: Identify, + identify: Identify, /// Information that we know about all nodes. nodes_info: FnvHashMap, /// Interval at which we perform garbage collection in `nodes_info`. @@ -63,7 +64,7 @@ struct NodeInfo { latest_ping: Option, } -impl DebugInfoBehaviour { +impl DebugInfoBehaviour { /// Builds a new `DebugInfoBehaviour`. pub fn new( user_agent: String, @@ -150,11 +151,10 @@ pub enum DebugInfoEvent { }, } -impl NetworkBehaviour for DebugInfoBehaviour -where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static { +impl NetworkBehaviour for DebugInfoBehaviour { type ProtocolsHandler = IntoProtocolsHandlerSelect< - as NetworkBehaviour>::ProtocolsHandler, - as NetworkBehaviour>::ProtocolsHandler + ::ProtocolsHandler, + ::ProtocolsHandler >; type OutEvent = DebugInfoEvent; diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index de49913b265e92498058028a8fd51902f06c93f5..8360fce5186074c075e17d6fdc78df83291618de 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -55,8 +55,6 @@ use libp2p::kad::record::{self, store::MemoryStore}; #[cfg(not(target_os = "unknown"))] use libp2p::{swarm::toggle::Toggle}; #[cfg(not(target_os = "unknown"))] -use libp2p::core::{nodes::Substream, muxing::StreamMuxerBox}; -#[cfg(not(target_os = "unknown"))] use libp2p::mdns::{Mdns, MdnsEvent}; use libp2p::multiaddr::Protocol; use log::{debug, info, trace, warn, error}; @@ -65,15 +63,15 @@ use std::task::{Context, Poll}; use sp_core::hexdisplay::HexDisplay; /// Implementation of `NetworkBehaviour` that discovers the nodes on the network. -pub struct DiscoveryBehaviour { +pub struct DiscoveryBehaviour { /// User-defined list of nodes and their addresses. Typically includes bootstrap nodes and /// reserved nodes. user_defined: Vec<(PeerId, Multiaddr)>, /// Kademlia requests and answers. - kademlia: Kademlia, + kademlia: Kademlia, /// Discovers nodes on the local network. #[cfg(not(target_os = "unknown"))] - mdns: Toggle>>, + mdns: Toggle, /// Stream that fires when we need to perform the next random Kademlia query. next_kad_random_query: Delay, /// After `next_kad_random_query` triggers, the next one triggers after this duration. @@ -87,9 +85,11 @@ pub struct DiscoveryBehaviour { /// If false, `addresses_of_peer` won't return any private IPv4 address, except for the ones /// stored in `user_defined`. allow_private_ipv4: bool, + /// Number of active connections over which we interrupt the discovery process. + discovery_only_if_under_num: u64, } -impl DiscoveryBehaviour { +impl DiscoveryBehaviour { /// Builds a new `DiscoveryBehaviour`. /// /// `user_defined` is a list of known address for nodes that never expire. @@ -98,6 +98,7 @@ impl DiscoveryBehaviour { user_defined: Vec<(PeerId, Multiaddr)>, enable_mdns: bool, allow_private_ipv4: bool, + discovery_only_if_under_num: u64, ) -> Self { if enable_mdns { #[cfg(target_os = "unknown")] @@ -120,6 +121,7 @@ impl DiscoveryBehaviour { local_peer_id: local_public_key.into_peer_id(), num_connections: 0, allow_private_ipv4, + discovery_only_if_under_num, #[cfg(not(target_os = "unknown"))] mdns: if enable_mdns { match Mdns::new() { @@ -190,7 +192,7 @@ pub enum DiscoveryOut { /// e.g. obtained through the `identify` protocol. UnroutablePeer(PeerId), - /// The DHT yeided results for the record request, grouped in (key, value) pairs. + /// The DHT yielded results for the record request, grouped in (key, value) pairs. ValueFound(Vec<(record::Key, Vec)>), /// The record requested was not found in the DHT. @@ -203,11 +205,8 @@ pub enum DiscoveryOut { ValuePutFailed(record::Key), } -impl NetworkBehaviour for DiscoveryBehaviour -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ - type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; +impl NetworkBehaviour for DiscoveryBehaviour { + type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; type OutEvent = DiscoveryOut; fn new_handler(&mut self) -> Self::ProtocolsHandler { @@ -331,11 +330,19 @@ where // Poll the stream that fires when we need to start a random Kademlia query. while let Poll::Ready(_) = self.next_kad_random_query.poll_unpin(cx) { - let random_peer_id = PeerId::random(); - debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ - {:?}", random_peer_id); + if self.num_connections < self.discovery_only_if_under_num { + let random_peer_id = PeerId::random(); + debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ + {:?}", random_peer_id); - self.kademlia.get_closest_peers(random_peer_id); + self.kademlia.get_closest_peers(random_peer_id); + } else { + debug!( + target: "sub-libp2p", + "Kademlia paused due to high number of connections ({})", + self.num_connections + ); + } // Schedule the next random query with exponentially increasing delay, // capped at 60 seconds. @@ -384,7 +391,14 @@ where DiscoveryOut::ValueFound(results) } + Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { + trace!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); + DiscoveryOut::ValueNotFound(e.into_key()) + } Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); DiscoveryOut::ValueNotFound(e.into_key()) } }; @@ -394,6 +408,8 @@ where let ev = match res { Ok(ok) => DiscoveryOut::ValuePut(ok.key), Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to put record: {:?}", e); DiscoveryOut::ValuePutFailed(e.into_key()) } }; @@ -435,6 +451,10 @@ where NetworkBehaviourAction::GenerateEvent(event) => { match event { MdnsEvent::Discovered(list) => { + if self.num_connections >= self.discovery_only_if_under_num { + continue; + } + self.discoveries.extend(list.into_iter().map(|(peer_id, _)| peer_id)); if let Some(peer_id) = self.discoveries.pop_front() { let ev = DiscoveryOut::Discovered(peer_id); @@ -502,7 +522,7 @@ mod tests { let user_defined = user_defined.clone(); let keypair_public = keypair.public(); async move { - DiscoveryBehaviour::new(keypair_public, user_defined, false, true).await + DiscoveryBehaviour::new(keypair_public, user_defined, false, true, 50).await } }); let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id()); diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index dd156360330acfa73f8115e027fc31910034ed81..a5397a4e3e6263f2c6c33263b7002549cedcacae 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -78,12 +78,8 @@ //! - DNS for addresses of the form `/dns4/example.com/tcp/5` or `/dns4/example.com/tcp/5/ws`. A //! node's address can contain a domain name. //! -//! The following encryption protocols are supported: -//! -//! - [Secio](https://github.com/libp2p/specs/tree/master/secio). A TLS-1.2-like protocol but -//! without certificates. Support for secio will likely be deprecated in the far future. -//! - [Noise](https://noiseprotocol.org/). Support for noise is very experimental. The details are -//! very blurry and may change at any moment. +//! On top of the base-layer protocol, the [Noise](https://noiseprotocol.org/) protocol is +//! negotiated and applied. The exact handshake protocol is experimental and is subject to change. //! //! The following multiplexing protocols are supported: //! @@ -140,10 +136,6 @@ //! - Light-client requests. When a light client requires information, a random node we have a //! substream open with is chosen, and the information is requested from it. //! - Gossiping. Used for example by grandpa. -//! - Network specialization. The network protocol can be specialized through a template parameter -//! of the network service. This specialization is free to send and receive messages with the -//! remote. This is meant to be used by the chain that is being built on top of Substrate -//! (eg. Polkadot). //! //! It is intended that in the future each of these components gets more isolated, so that they //! are free to open and close their own substreams, and so that syncing and light client requests @@ -160,7 +152,8 @@ //! //! After the `NetworkWorker` has been created, the important things to do are: //! -//! - Calling `NetworkWorker::poll` in order to advance the network. +//! - Calling `NetworkWorker::poll` in order to advance the network. This can be done by +//! dispatching a background task with the [`NetworkWorker`]. //! - Calling `on_block_import` whenever a block is added to the client. //! - Calling `on_block_finalized` whenever a block is finalized. //! - Calling `trigger_repropagate` when a transaction is added to the pool. @@ -180,34 +173,27 @@ mod utils; pub mod config; pub mod error; +pub mod network_state; -pub use chain::{Client as ClientHandle, FinalityProofProvider}; -pub use service::{ - NetworkService, NetworkWorker, TransactionPool, ExHashT, ReportHandle, - NetworkStateInfo, -}; -pub use protocol::{PeerInfo, Context, ProtocolConfig, message, specialization}; +pub use service::{NetworkService, NetworkStateInfo, NetworkWorker, ExHashT, ReportHandle}; +pub use protocol::PeerInfo; pub use protocol::event::{Event, DhtEvent}; pub use protocol::sync::SyncState; pub use libp2p::{Multiaddr, PeerId}; #[doc(inline)] pub use libp2p::multiaddr; -pub use message::{generic as generic_message, RequestId, Status as StatusMessage}; -pub use on_demand_layer::{OnDemand, RemoteResponse}; -pub use sc_peerset::ReputationChange; - -// Used by the `construct_simple_protocol!` macro. +// Note: these re-exports shouldn't be part of the public API of the crate and will be removed in +// the future. +#[doc(hidden)] +pub use protocol::message; #[doc(hidden)] -pub use sp_runtime::traits::Block as BlockT; +pub use protocol::message::Status as StatusMessage; -use libp2p::core::ConnectedPoint; -use serde::{Deserialize, Serialize}; -use slog_derive::SerdeValue; -use std::{collections::{HashMap, HashSet}, time::Duration}; +pub use sc_peerset::ReputationChange; /// Extension trait for `NetworkBehaviour` that also accepts discovering nodes. -pub trait DiscoveryNetBehaviour { +trait DiscoveryNetBehaviour { /// Notify the protocol that we have learned about the existence of nodes. /// /// Can (or most likely will) be called multiple times with the same `PeerId`s. @@ -216,90 +202,3 @@ pub trait DiscoveryNetBehaviour { /// system, or remove nodes that will fail to reach. fn add_discovered_nodes(&mut self, nodes: impl Iterator); } - -/// Returns general information about the networking. -/// -/// Meant for general diagnostic purposes. -/// -/// **Warning**: This API is not stable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, SerdeValue)] -#[serde(rename_all = "camelCase")] -pub struct NetworkState { - /// PeerId of the local node. - pub peer_id: String, - /// List of addresses the node is currently listening on. - pub listened_addresses: HashSet, - /// List of addresses the node knows it can be reached as. - pub external_addresses: HashSet, - /// List of node we're connected to. - pub connected_peers: HashMap, - /// List of node that we know of but that we're not connected to. - pub not_connected_peers: HashMap, - /// Downloaded bytes per second averaged over the past few seconds. - pub average_download_per_sec: u64, - /// Uploaded bytes per second averaged over the past few seconds. - pub average_upload_per_sec: u64, - /// State of the peerset manager. - pub peerset: serde_json::Value, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NetworkStatePeer { - /// How we are connected to the node. - pub endpoint: NetworkStatePeerEndpoint, - /// Node information, as provided by the node itself. Can be empty if not known yet. - pub version_string: Option, - /// Latest ping duration with this node. - pub latest_ping_time: Option, - /// If true, the peer is "enabled", which means that we try to open Substrate-related protocols - /// with this peer. If false, we stick to Kademlia and/or other network-only protocols. - pub enabled: bool, - /// If true, the peer is "open", which means that we have a Substrate-related protocol - /// with this peer. - pub open: bool, - /// List of addresses known for this node. - pub known_addresses: HashSet, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NetworkStateNotConnectedPeer { - /// List of addresses known for this node. - pub known_addresses: HashSet, - /// Node information, as provided by the node itself, if we were ever connected to this node. - pub version_string: Option, - /// Latest ping duration with this node, if we were ever connected to this node. - pub latest_ping_time: Option, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum NetworkStatePeerEndpoint { - /// We are dialing the given address. - Dialing(Multiaddr), - /// We are listening. - Listening { - /// Local address of the connection. - local_addr: Multiaddr, - /// Address data is sent back to. - send_back_addr: Multiaddr, - }, -} - -impl From for NetworkStatePeerEndpoint { - fn from(endpoint: ConnectedPoint) -> Self { - match endpoint { - ConnectedPoint::Dialer { address } => - NetworkStatePeerEndpoint::Dialing(address), - ConnectedPoint::Listener { local_addr, send_back_addr } => - NetworkStatePeerEndpoint::Listening { - local_addr, - send_back_addr - } - } - } -} diff --git a/client/network/src/network_state.rs b/client/network/src/network_state.rs new file mode 100644 index 0000000000000000000000000000000000000000..00d53976ae8fc8960e9045b1dbab7d1621429f26 --- /dev/null +++ b/client/network/src/network_state.rs @@ -0,0 +1,111 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Information about the networking, for diagnostic purposes. +//! +//! **Warning**: These APIs are not stable. + +use libp2p::{core::ConnectedPoint, Multiaddr}; +use serde::{Deserialize, Serialize}; +use slog_derive::SerdeValue; +use std::{collections::{HashMap, HashSet}, time::Duration}; + +/// Returns general information about the networking. +/// +/// Meant for general diagnostic purposes. +/// +/// **Warning**: This API is not stable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, SerdeValue)] +#[serde(rename_all = "camelCase")] +pub struct NetworkState { + /// PeerId of the local node. + pub peer_id: String, + /// List of addresses the node is currently listening on. + pub listened_addresses: HashSet, + /// List of addresses the node knows it can be reached as. + pub external_addresses: HashSet, + /// List of node we're connected to. + pub connected_peers: HashMap, + /// List of node that we know of but that we're not connected to. + pub not_connected_peers: HashMap, + /// Downloaded bytes per second averaged over the past few seconds. + pub average_download_per_sec: u64, + /// Uploaded bytes per second averaged over the past few seconds. + pub average_upload_per_sec: u64, + /// State of the peerset manager. + pub peerset: serde_json::Value, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Peer { + /// How we are connected to the node. + pub endpoint: PeerEndpoint, + /// Node information, as provided by the node itself. Can be empty if not known yet. + pub version_string: Option, + /// Latest ping duration with this node. + pub latest_ping_time: Option, + /// If true, the peer is "enabled", which means that we try to open Substrate-related protocols + /// with this peer. If false, we stick to Kademlia and/or other network-only protocols. + pub enabled: bool, + /// If true, the peer is "open", which means that we have a Substrate-related protocol + /// with this peer. + pub open: bool, + /// List of addresses known for this node. + pub known_addresses: HashSet, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotConnectedPeer { + /// List of addresses known for this node. + pub known_addresses: HashSet, + /// Node information, as provided by the node itself, if we were ever connected to this node. + pub version_string: Option, + /// Latest ping duration with this node, if we were ever connected to this node. + pub latest_ping_time: Option, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PeerEndpoint { + /// We are dialing the given address. + Dialing(Multiaddr), + /// We are listening. + Listening { + /// Local address of the connection. + local_addr: Multiaddr, + /// Address data is sent back to. + send_back_addr: Multiaddr, + }, +} + +impl From for PeerEndpoint { + fn from(endpoint: ConnectedPoint) -> Self { + match endpoint { + ConnectedPoint::Dialer { address } => + PeerEndpoint::Dialing(address), + ConnectedPoint::Listener { local_addr, send_back_addr } => + PeerEndpoint::Listening { + local_addr, + send_back_addr + } + } + } +} diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 5e8df2831ba635973ef396d65bbb055cf33bb87e..d344321e68dd0440cfcd2dcc808d8bd062fc0c44 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -15,12 +15,12 @@ // along with Substrate. If not, see . use crate::{DiscoveryNetBehaviour, config::ProtocolId}; -use legacy_proto::{LegacyProto, LegacyProtoOut}; use crate::utils::interval; use bytes::{Bytes, BytesMut}; use futures::prelude::*; +use generic_proto::{GenericProto, GenericProtoOut}; use libp2p::{Multiaddr, PeerId}; -use libp2p::core::{ConnectedPoint, nodes::{listeners::ListenerId, Substream}, muxing::StreamMuxerBox}; +use libp2p::core::{ConnectedPoint, nodes::listeners::ListenerId}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use sp_core::storage::{StorageKey, ChildInfo}; @@ -36,13 +36,13 @@ use sp_runtime::traits::{ }; use sp_arithmetic::traits::SaturatedConversion; use message::{BlockAnnounce, BlockAttributes, Direction, FromBlock, Message, RequestId}; -use message::generic::{Message as GenericMessage, ConsensusMessage}; +use message::generic::Message as GenericMessage; use light_dispatch::{LightDispatch, LightDispatchNetwork, RequestData}; -use specialization::NetworkSpecialization; use sync::{ChainSync, SyncState}; use crate::service::{TransactionPool, ExHashT}; use crate::config::{BoxFinalityProofRequestBuilder, Roles}; use rustc_hex::ToHex; +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::fmt::Write; @@ -52,20 +52,35 @@ use crate::chain::{Client, FinalityProofProvider}; use sc_client_api::{FetchChecker, ChangesProof, StorageProof}; use crate::error; use util::LruHashSet; +use wasm_timer::Instant; + +// Include sources generated from protobuf definitions. +pub mod api { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/api.v1.rs")); + pub mod light { + include!(concat!(env!("OUT_DIR"), "/api.v1.light.rs")); + } + } +} -mod legacy_proto; +mod generic_proto; mod util; +pub mod block_requests; pub mod message; pub mod event; +pub mod light_client_handler; pub mod light_dispatch; -pub mod specialization; pub mod sync; +pub use block_requests::BlockRequests; +pub use light_client_handler::LightClientHandler; + const REQUEST_TIMEOUT_SEC: u64 = 40; /// Interval at which we perform time based maintenance const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100); -/// Interval at which we propagate exstrinsics; +/// Interval at which we propagate extrinsics; const PROPAGATE_TIMEOUT: time::Duration = time::Duration::from_millis(2900); /// Maximim number of known block hashes to keep for a peer. @@ -81,7 +96,7 @@ pub(crate) const MIN_VERSION: u32 = 3; // Maximum allowed entries in `BlockResponse` const MAX_BLOCK_DATA_RESPONSE: u32 = 128; /// When light node connects to the full node and the full node is behind light node -/// for at least `LIGHT_MAXIMAL_BLOCKS_DIFFERENCE` blocks, we consider it unuseful +/// for at least `LIGHT_MAXIMAL_BLOCKS_DIFFERENCE` blocks, we consider it not useful /// and disconnect to free connection slot. const LIGHT_MAXIMAL_BLOCKS_DIFFERENCE: u64 = 8192; @@ -116,10 +131,12 @@ mod rep { pub const BAD_PROTOCOL: Rep = Rep::new_fatal("Unsupported protocol"); /// Peer role does not match (e.g. light peer connecting to another light peer). pub const BAD_ROLE: Rep = Rep::new_fatal("Unsupported role"); + /// Peer response data does not have requested bits. + pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); } // Lock must always be taken in order declared here. -pub struct Protocol, H: ExHashT> { +pub struct Protocol { /// Interval at which we call `tick`. tick_timeout: Pin + Send>>, /// Interval at which we call `propagate_extrinsics`. @@ -129,7 +146,6 @@ pub struct Protocol, H: ExHashT> { light_dispatch: LightDispatch, genesis_hash: B::Hash, sync: ChainSync, - specialization: S, context_data: ContextData, /// List of nodes for which we perform additional logging because they are important for the /// user. @@ -142,9 +158,11 @@ pub struct Protocol, H: ExHashT> { /// When asked for a proof of finality, we use this struct to build one. finality_proof_provider: Option>>, /// Handles opening the unique substream and sending and receiving raw messages. - behaviour: LegacyProto>, - /// List of notification protocols that have been registered. - registered_notif_protocols: HashSet, + behaviour: GenericProto, + /// For each legacy gossiping engine ID, the corresponding new protocol name. + protocol_name_by_engine: HashMap>, + /// For each protocol name, the legacy gossiping engine ID. + protocol_engine_by_name: HashMap, ConsensusEngineId>, } #[derive(Default)] @@ -158,7 +176,7 @@ struct PacketStats { /// A peer that we are connected to /// and from whom we have not yet received a Status message. struct HandshakingPeer { - timestamp: time::Instant, + timestamp: Instant, } /// Peer information @@ -166,9 +184,9 @@ struct HandshakingPeer { struct Peer { info: PeerInfo, /// Current block request, if any. - block_request: Option<(time::Instant, message::BlockRequest)>, - /// Requests we are no longer insterested in. - obsolete_requests: HashMap, + block_request: Option<(Instant, message::BlockRequest)>, + /// Requests we are no longer interested in. + obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. known_extrinsics: LruHashSet, /// Holds a set of blocks known to this peer. @@ -191,7 +209,7 @@ pub struct PeerInfo { } struct LightDispatchIn<'a> { - behaviour: &'a mut LegacyProto>, + behaviour: &'a mut GenericProto, peerset: sc_peerset::PeersetHandle, } @@ -316,55 +334,6 @@ impl<'a, B: BlockT> LightDispatchNetwork for LightDispatchIn<'a> { } } -/// Context for a network-specific handler. -pub trait Context { - /// Adjusts the reputation of the peer. Use this to point out that a peer has been malign or - /// irresponsible or appeared lazy. - fn report_peer(&mut self, who: PeerId, reputation: sc_peerset::ReputationChange); - - /// Force disconnecting from a peer. Use this when a peer misbehaved. - fn disconnect_peer(&mut self, who: PeerId); - - /// Send a chain-specific message to a peer. - fn send_chain_specific(&mut self, who: PeerId, message: Vec); -} - -/// Protocol context. -struct ProtocolContext<'a, B: 'a + BlockT, H: 'a + ExHashT> { - behaviour: &'a mut LegacyProto>, - context_data: &'a mut ContextData, - peerset_handle: &'a sc_peerset::PeersetHandle, -} - -impl<'a, B: BlockT + 'a, H: 'a + ExHashT> ProtocolContext<'a, B, H> { - fn new( - context_data: &'a mut ContextData, - behaviour: &'a mut LegacyProto>, - peerset_handle: &'a sc_peerset::PeersetHandle, - ) -> Self { - ProtocolContext { context_data, peerset_handle, behaviour } - } -} - -impl<'a, B: BlockT + 'a, H: ExHashT + 'a> Context for ProtocolContext<'a, B, H> { - fn report_peer(&mut self, who: PeerId, reputation: sc_peerset::ReputationChange) { - self.peerset_handle.report_peer(who, reputation) - } - - fn disconnect_peer(&mut self, who: PeerId) { - self.behaviour.disconnect_peer(&who) - } - - fn send_chain_specific(&mut self, who: PeerId, message: Vec) { - send_message:: ( - self.behaviour, - &mut self.context_data.stats, - &who, - GenericMessage::ChainSpecific(message) - ) - } -} - /// Data necessary to create a context. struct ContextData { // All connected peers @@ -391,20 +360,19 @@ impl Default for ProtocolConfig { } } -impl, H: ExHashT> Protocol { +impl Protocol { /// Create a new instance. pub fn new( config: ProtocolConfig, chain: Arc>, checker: Arc>, - specialization: S, transaction_pool: Arc>, finality_proof_provider: Option>>, finality_proof_request_builder: Option>, protocol_id: ProtocolId, peerset_config: sc_peerset::PeersetConfig, block_announce_validator: Box + Send> - ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { + ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { let info = chain.info(); let sync = ChainSync::new( config.roles, @@ -426,7 +394,7 @@ impl, H: ExHashT> Protocol { let (peerset, peerset_handle) = sc_peerset::Peerset::from_config(peerset_config); let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::>(); - let behaviour = LegacyProto::new(protocol_id, versions, peerset); + let behaviour = GenericProto::new(protocol_id, versions, peerset); let protocol = Protocol { tick_timeout: Box::pin(interval(TICK_TIMEOUT)), @@ -440,14 +408,14 @@ impl, H: ExHashT> Protocol { light_dispatch: LightDispatch::new(checker), genesis_hash: info.genesis_hash, sync, - specialization, handshaking_peers: HashMap::new(), important_peers, transaction_pool, finality_proof_provider, peerset_handle: peerset_handle.clone(), behaviour, - registered_notif_protocols: HashSet::new(), + protocol_name_by_engine: HashMap::new(), + protocol_engine_by_name: HashMap::new(), }; Ok((protocol, peerset_handle)) @@ -630,7 +598,7 @@ impl, H: ExHashT> Protocol { GenericMessage::RemoteReadChildRequest(request) => self.on_remote_read_child_request(who, request), GenericMessage::Consensus(msg) => - return if self.registered_notif_protocols.contains(&msg.engine_id) { + return if self.protocol_name_by_engine.contains_key(&msg.engine_id) { CustomMessageOutcome::NotificationsReceived { remote: who.clone(), messages: vec![(msg.engine_id, From::from(msg.data))], @@ -643,7 +611,7 @@ impl, H: ExHashT> Protocol { let messages = messages .into_iter() .filter_map(|msg| { - if self.registered_notif_protocols.contains(&msg.engine_id) { + if self.protocol_name_by_engine.contains_key(&msg.engine_id) { Some((msg.engine_id, From::from(msg.data))) } else { warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id); @@ -661,11 +629,6 @@ impl, H: ExHashT> Protocol { CustomMessageOutcome::None }; }, - GenericMessage::ChainSpecific(msg) => self.specialization.on_message( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle), - who, - msg, - ), } CustomMessageOutcome::None @@ -690,18 +653,10 @@ impl, H: ExHashT> Protocol { ); } - /// Locks `self` and returns a context plus the network specialization. - pub fn specialization_lock<'a>( - &'a mut self, - ) -> (impl Context + 'a, &'a mut S) { - let context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); - (context, &mut self.specialization) - } - /// Called when a new peer is connected pub fn on_peer_connected(&mut self, who: PeerId) { trace!(target: "sync", "Connecting {}", who); - self.handshaking_peers.insert(who.clone(), HandshakingPeer { timestamp: time::Instant::now() }); + self.handshaking_peers.insert(who.clone(), HandshakingPeer { timestamp: Instant::now() }); self.send_status(who); } @@ -719,9 +674,7 @@ impl, H: ExHashT> Protocol { self.context_data.peers.remove(&peer) }; if let Some(_peer_data) = removed { - let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); self.sync.peer_disconnected(peer.clone()); - self.specialization.on_disconnect(&mut context, peer.clone()); self.light_dispatch.on_disconnect(LightDispatchIn { behaviour: &mut self.behaviour, peerset: self.peerset_handle.clone(), @@ -750,12 +703,14 @@ impl, H: ExHashT> Protocol { peer: PeerId, request: message::BlockRequest ) { - trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?}", + trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?} for {:?}", request.id, peer, request.from, request.to, - request.max); + request.max, + request.fields, + ); // sending block requests to the node that is unable to serve it is considered a bad behavior if !self.config.roles.is_full() { @@ -803,6 +758,11 @@ impl, H: ExHashT> Protocol { message_queue: None, justification, }; + // Stop if we don't have requested block body + if get_body && block_data.body.is_none() { + trace!(target: "sync", "Missing data for block request."); + break; + } blocks.push(block_data); match request.direction { message::Direction::Ascending => id = BlockId::Number(number + One::one()), @@ -833,7 +793,7 @@ impl, H: ExHashT> Protocol { request: message::BlockRequest, response: message::BlockResponse, ) -> CustomMessageOutcome { - let blocks_range = match ( + let blocks_range = || match ( response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), ) { @@ -845,11 +805,9 @@ impl, H: ExHashT> Protocol { response.id, peer, response.blocks.len(), - blocks_range + blocks_range(), ); - // TODO [andre]: move this logic to the import queue so that - // justifications are imported asynchronously (#1482) if request.fields == message::BlockAttributes::JUSTIFICATION { match self.sync.on_block_justification(peer, response) { Ok(sync::OnBlockJustification::Nothing) => CustomMessageOutcome::None, @@ -862,6 +820,20 @@ impl, H: ExHashT> Protocol { } } } else { + // Validate fields against the request. + if request.fields.contains(message::BlockAttributes::HEADER) && response.blocks.iter().any(|b| b.header.is_none()) { + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing header for a block"); + return CustomMessageOutcome::None + } + if request.fields.contains(message::BlockAttributes::BODY) && response.blocks.iter().any(|b| b.body.is_none()) { + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing body for a block"); + return CustomMessageOutcome::None + } + match self.sync.on_block_data(peer, Some(request), response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), @@ -890,21 +862,21 @@ impl, H: ExHashT> Protocol { } fn maintain_peers(&mut self) { - let tick = time::Instant::now(); + let tick = Instant::now(); let mut aborting = Vec::new(); { for (who, peer) in self.context_data.peers.iter() { if peer.block_request.as_ref().map_or(false, |(t, _)| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { log!( target: "sync", - if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace }, + if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, "Request timeout {}", who ); aborting.push(who.clone()); } else if peer.obsolete_requests.values().any(|t| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { log!( target: "sync", - if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace }, + if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, "Obsolete timeout {}", who ); aborting.push(who.clone()); @@ -915,16 +887,13 @@ impl, H: ExHashT> Protocol { { log!( target: "sync", - if self.important_peers.contains(&who) { Level::Warn } else { Level::Trace }, + if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, "Handshake timeout {}", who ); aborting.push(who.clone()); } } - self.specialization.maintain_peers( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle) - ); for p in aborting { self.behaviour.disconnect_peer(&p); self.peerset_handle.report_peer(p, rep::TIMEOUT); @@ -1040,13 +1009,10 @@ impl, H: ExHashT> Protocol { } } - let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); - self.specialization.on_connect(&mut context, who.clone(), status); - // Notify all the notification protocols as open. CustomMessageOutcome::NotificationStreamOpened { remote: who, - protocols: self.registered_notif_protocols.iter().cloned().collect(), + protocols: self.protocol_name_by_engine.keys().cloned().collect(), roles: info.roles, } } @@ -1061,18 +1027,15 @@ impl, H: ExHashT> Protocol { engine_id: ConsensusEngineId, message: impl Into> ) { - if !self.registered_notif_protocols.contains(&engine_id) { + if let Some(protocol_name) = self.protocol_name_by_engine.get(&engine_id) { + self.behaviour.write_notification(&target, engine_id, protocol_name.clone(), message); + } else { error!( target: "sub-libp2p", "Sending a notification with a protocol that wasn't registered: {:?}", engine_id ); } - - self.send_message(&target, GenericMessage::Consensus(ConsensusMessage { - engine_id, - data: message.into(), - })); } /// Registers a new notifications protocol. @@ -1082,9 +1045,14 @@ impl, H: ExHashT> Protocol { pub fn register_notifications_protocol( &mut self, engine_id: ConsensusEngineId, + protocol_name: impl Into>, ) -> Vec { - if !self.registered_notif_protocols.insert(engine_id) { - error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", engine_id); + let protocol_name = protocol_name.into(); + if self.protocol_name_by_engine.insert(engine_id, protocol_name.clone()).is_some() { + error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", protocol_name); + } else { + self.behaviour.register_notif_protocol(protocol_name.clone(), engine_id, Vec::new()); + self.protocol_engine_by_name.insert(protocol_name, engine_id); } // Registering a protocol while we already have open connections isn't great, but for now @@ -1272,7 +1240,7 @@ impl, H: ExHashT> Protocol { roles: self.config.roles.into(), best_number: info.best_number, best_hash: info.best_hash, - chain_status: self.specialization.status(), + chain_status: Vec::new(), // TODO: find a way to make this backwards-compatible }; self.send_message(&who, GenericMessage::Status(status)) @@ -1341,15 +1309,10 @@ impl, H: ExHashT> Protocol { /// Call this when a block has been imported in the import queue and we should announce it on /// the network. - pub fn on_block_imported(&mut self, hash: B::Hash, header: &B::Header, data: Vec, is_best: bool) { + pub fn on_block_imported(&mut self, header: &B::Header, data: Vec, is_best: bool) { if is_best { self.sync.update_chain_info(header); } - self.specialization.on_block_imported( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle), - hash.clone(), - header, - ); // blocks are not announced by light clients if self.config.roles.is_light() { @@ -1819,7 +1782,7 @@ pub enum CustomMessageOutcome { } fn send_request( - behaviour: &mut LegacyProto>, + behaviour: &mut GenericProto, stats: &mut HashMap<&'static str, PacketStats>, peers: &mut HashMap>, who: &PeerId, @@ -1833,14 +1796,14 @@ fn send_request( trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); peer.obsolete_requests.insert(request.id, timestamp); } - peer.block_request = Some((time::Instant::now(), r.clone())); + peer.block_request = Some((Instant::now(), r.clone())); } } send_message::(behaviour, stats, who, message) } fn send_message( - behaviour: &mut LegacyProto>, + behaviour: &mut GenericProto, stats: &mut HashMap<&'static str, PacketStats>, who: &PeerId, message: Message, @@ -1852,9 +1815,8 @@ fn send_message( behaviour.send_packet(who, encoded); } -impl, H: ExHashT> NetworkBehaviour for -Protocol { - type ProtocolsHandler = > as NetworkBehaviour>::ProtocolsHandler; +impl NetworkBehaviour for Protocol { + type ProtocolsHandler = ::ProtocolsHandler; type OutEvent = CustomMessageOutcome; fn new_handler(&mut self) -> Self::ProtocolsHandler { @@ -1940,25 +1902,21 @@ Protocol { }; let outcome = match event { - LegacyProtoOut::CustomProtocolOpen { peer_id, version, .. } => { - debug_assert!( - version <= CURRENT_VERSION as u8 - && version >= MIN_VERSION as u8 - ); + GenericProtoOut::CustomProtocolOpen { peer_id, .. } => { self.on_peer_connected(peer_id.clone()); CustomMessageOutcome::None } - LegacyProtoOut::CustomProtocolClosed { peer_id, .. } => { + GenericProtoOut::CustomProtocolClosed { peer_id, .. } => { self.on_peer_disconnected(peer_id.clone()); // Notify all the notification protocols as closed. CustomMessageOutcome::NotificationStreamClosed { remote: peer_id, - protocols: self.registered_notif_protocols.iter().cloned().collect(), + protocols: self.protocol_name_by_engine.keys().cloned().collect(), } }, - LegacyProtoOut::CustomMessage { peer_id, message } => + GenericProtoOut::CustomMessage { peer_id, message } => self.on_custom_message(peer_id, message), - LegacyProtoOut::Clogged { peer_id, messages } => { + GenericProtoOut::Clogged { peer_id, messages } => { debug!(target: "sync", "{} clogging messages:", messages.len()); for msg in messages.into_iter().take(5) { let message: Option> = Decode::decode(&mut &msg[..]).ok(); @@ -2014,13 +1972,13 @@ Protocol { } } -impl, H: ExHashT> DiscoveryNetBehaviour for Protocol { +impl DiscoveryNetBehaviour for Protocol { fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { self.behaviour.add_discovered_nodes(peer_ids) } } -impl, H: ExHashT> Drop for Protocol { +impl Drop for Protocol { fn drop(&mut self) { debug!(target: "sync", "Network stats:\n{}", self.format_stats()); } diff --git a/client/network/src/protocol/block_requests.rs b/client/network/src/protocol/block_requests.rs new file mode 100644 index 0000000000000000000000000000000000000000..ef970657c5ff9477bc939b6dd98963b4e41387e9 --- /dev/null +++ b/client/network/src/protocol/block_requests.rs @@ -0,0 +1,356 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! `NetworkBehaviour` implementation which handles incoming block requests. +//! +//! Every request is coming in on a separate connection substream which gets +//! closed after we have sent the response back. Incoming requests are encoded +//! as protocol buffers (cf. `api.v1.proto`). + +#![allow(unused)] + +use bytes::Bytes; +use codec::{Encode, Decode}; +use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, message::BlockAttributes} +}; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use libp2p::{ + core::{ + ConnectedPoint, + Multiaddr, + PeerId, + upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + upgrade::{DeniedUpgrade, read_one, write_one} + }, + swarm::{ + NegotiatedSubstream, + NetworkBehaviour, + NetworkBehaviourAction, + OneShotHandler, + PollParameters, + SubstreamProtocol + } +}; +use prost::Message; +use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; +use std::{ + cmp::min, + io, + iter, + sync::Arc, + time::Duration, + task::{Context, Poll} +}; +use void::{Void, unreachable}; + +// Type alias for convenience. +pub type Error = Box; + +/// Configuration options for `BlockRequests`. +#[derive(Debug, Clone)] +pub struct Config { + max_block_data_response: u32, + max_request_len: usize, + inactivity_timeout: Duration, + protocol: Bytes, +} + +impl Config { + /// Create a fresh configuration with the following options: + /// + /// - max. block data in response = 128 + /// - max. request size = 1 MiB + /// - inactivity timeout = 15s + pub fn new(id: &ProtocolId) -> Self { + let mut c = Config { + max_block_data_response: 128, + max_request_len: 1024 * 1024, + inactivity_timeout: Duration::from_secs(15), + protocol: Bytes::new(), + }; + c.set_protocol(id); + c + } + + /// Limit the max. number of block data in a response. + pub fn set_max_block_data_response(&mut self, v: u32) -> &mut Self { + self.max_block_data_response = v; + self + } + + /// Limit the max. length of incoming block request bytes. + pub fn set_max_request_len(&mut self, v: usize) -> &mut Self { + self.max_request_len = v; + self + } + + /// Limit the max. duration the substream may remain inactive before closing it. + pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { + self.inactivity_timeout = v; + self + } + + /// Set protocol to use for upgrade negotiation. + pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { + let mut v = Vec::new(); + v.extend_from_slice(b"/"); + v.extend_from_slice(id.as_bytes()); + v.extend_from_slice(b"/sync/1"); + self.protocol = v.into(); + self + } +} + +/// The block request handling behaviour. +pub struct BlockRequests { + /// This behaviour's configuration. + config: Config, + /// Blockchain client. + chain: Arc>, + /// Futures sending back the block request response. + outgoing: FuturesUnordered>, +} + +impl BlockRequests +where + B: Block, +{ + pub fn new(cfg: Config, chain: Arc>) -> Self { + BlockRequests { + config: cfg, + chain, + outgoing: FuturesUnordered::new(), + } + } + + /// Callback, invoked when a new block request has been received from remote. + fn on_block_request + ( &mut self + , peer: &PeerId + , request: &api::v1::BlockRequest + ) -> Result + { + log::trace!("block request {} from peer {}: from block {:?} to block {:?}, max blocks {:?}", + request.id, + peer, + request.from_block, + request.to_block, + request.max_blocks); + + let from_block_id = + match request.from_block { + Some(api::v1::block_request::FromBlock::Hash(ref h)) => { + let h = Decode::decode(&mut h.as_ref())?; + BlockId::::Hash(h) + } + Some(api::v1::block_request::FromBlock::Number(ref n)) => { + let n = Decode::decode(&mut n.as_ref())?; + BlockId::::Number(n) + } + None => { + let msg = "missing `BlockRequest::from_block` field"; + return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + } + }; + + let max_blocks = + if request.max_blocks == 0 { + self.config.max_block_data_response + } else { + min(request.max_blocks, self.config.max_block_data_response) + }; + + let direction = + if request.direction == api::v1::Direction::Ascending as i32 { + api::v1::Direction::Ascending + } else if request.direction == api::v1::Direction::Descending as i32 { + api::v1::Direction::Descending + } else { + let msg = format!("invalid `BlockRequest::direction` value: {}", request.direction); + return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + }; + + let attributes = BlockAttributes::decode(&mut request.fields.to_be_bytes().as_ref())?; + let get_header = attributes.contains(BlockAttributes::HEADER); + let get_body = attributes.contains(BlockAttributes::BODY); + let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); + + let mut blocks = Vec::new(); + let mut block_id = from_block_id; + while let Some(header) = self.chain.header(&block_id).unwrap_or(None) { + if blocks.len() >= max_blocks as usize { + break + } + + let number = header.number().clone(); + let hash = header.hash(); + let parent_hash = header.parent_hash().clone(); + + let block_data = api::v1::BlockData { + hash: hash.encode(), + header: if get_header { + header.encode() + } else { + Vec::new() + }, + body: if get_body { + self.chain.body(&BlockId::Hash(hash))? + .unwrap_or(Vec::new()) + .iter_mut() + .map(|extrinsic| extrinsic.encode()) + .collect() + } else { + Vec::new() + }, + receipt: Vec::new(), + message_queue: Vec::new(), + justification: if get_justification { + self.chain.justification(&BlockId::Hash(hash))?.unwrap_or(Vec::new()) + } else { + Vec::new() + } + }; + + blocks.push(block_data); + + match direction { + api::v1::Direction::Ascending => { + block_id = BlockId::Number(number + One::one()) + } + api::v1::Direction::Descending => { + if number.is_zero() { + break + } + block_id = BlockId::Hash(parent_hash) + } + } + } + + Ok(api::v1::BlockResponse { id: request.id, blocks }) + } +} + +impl NetworkBehaviour for BlockRequests +where + B: Block +{ + type ProtocolsHandler = OneShotHandler>; + type OutEvent = Void; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + let p = Protocol { + max_request_len: self.config.max_request_len, + protocol: self.config.protocol.clone(), + }; + OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + } + + fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { + Vec::new() + } + + fn inject_connected(&mut self, _peer: PeerId, _info: ConnectedPoint) { + } + + fn inject_disconnected(&mut self, _peer: &PeerId, _info: ConnectedPoint) { + } + + fn inject_node_event(&mut self, peer: PeerId, Request(request, mut stream): Request) { + match self.on_block_request(&peer, &request) { + Ok(res) => { + log::trace!("enqueueing block response {} for peer {} with {} blocks", res.id, peer, res.blocks.len()); + let mut data = Vec::with_capacity(res.encoded_len()); + if let Err(e) = res.encode(&mut data) { + log::debug!("error encoding block response {} for peer {}: {}", res.id, peer, e) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!("error writing block response: {}", e) + } + }; + self.outgoing.push(future.boxed()) + } + } + Err(e) => log::debug!("error handling block request {} from peer {}: {}", request.id, peer, e) + } + } + + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) -> Poll> { + while let Poll::Ready(Some(_)) = self.outgoing.poll_next_unpin(cx) {} + Poll::Pending + } +} + +/// The incoming block request. +/// +/// Holds the protobuf value and the connection substream which made the +/// request and over which to send the response. +#[derive(Debug)] +pub struct Request(api::v1::BlockRequest, T); + +impl From for Request { + fn from(v: Void) -> Self { + unreachable(v) + } +} + +/// Substream upgrade protocol. +/// +/// We attempt to parse an incoming protobuf encoded request (cf. `Request`) +/// which will be handled by the `BlockRequests` behaviour, i.e. the request +/// will become visible via `inject_node_event` which then dispatches to the +/// relevant callback to process the message and prepare a response. +#[derive(Debug, Clone)] +pub struct Protocol { + /// The max. request length in bytes. + max_request_len: usize, + /// The protocol to use during upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for Protocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl InboundUpgrade for Protocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Request; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + let len = self.max_request_len; + let vec = read_one(&mut s, len).await?; + match api::v1::BlockRequest::decode(&vec[..]) { + Ok(r) => Ok(Request(r, s)), + Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) + } + }; + future.boxed() + } +} + diff --git a/client/network/src/protocol/event.rs b/client/network/src/protocol/event.rs index e239e9d9831821d524434d311a26ed47d02d0d4d..78490863be9c250ad78605f634c95670ab8bf9ce 100644 --- a/client/network/src/protocol/event.rs +++ b/client/network/src/protocol/event.rs @@ -36,7 +36,7 @@ pub enum DhtEvent { /// The record has been successfully inserted into the DHT. ValuePut(Key), - /// An error has occured while putting a record into the DHT. + /// An error has occurred while putting a record into the DHT. ValuePutFailed(Key), } diff --git a/client/network/src/protocol/legacy_proto.rs b/client/network/src/protocol/generic_proto.rs similarity index 86% rename from client/network/src/protocol/legacy_proto.rs rename to client/network/src/protocol/generic_proto.rs index 434782f7d5065de6372fa651f1ed1ac29b0209aa..f703287f386fdcf9aebcfc8d3fa3d0c5971c7eff 100644 --- a/client/network/src/protocol/legacy_proto.rs +++ b/client/network/src/protocol/generic_proto.rs @@ -17,10 +17,10 @@ //! Implementation of libp2p's `NetworkBehaviour` trait that opens a single substream with the //! remote and then allows any communication with them. //! -//! The `Protocol` struct uses `LegacyProto` in order to open substreams with the rest of the +//! The `Protocol` struct uses `GenericProto` in order to open substreams with the rest of the //! network, then performs the Substrate protocol handling on top. -pub use self::behaviour::{LegacyProto, LegacyProtoOut}; +pub use self::behaviour::{GenericProto, GenericProtoOut}; mod behaviour; mod handler; diff --git a/client/network/src/protocol/legacy_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs similarity index 85% rename from client/network/src/protocol/legacy_proto/behaviour.rs rename to client/network/src/protocol/generic_proto/behaviour.rs index b4047f320ac2a2228cbedde7b6f7fcef41f8440a..24e96681a084afcb71b9f41b5f78670a384bc80e 100644 --- a/client/network/src/protocol/legacy_proto/behaviour.rs +++ b/client/network/src/protocol/generic_proto/behaviour.rs @@ -15,9 +15,12 @@ // along with Substrate. If not, see . use crate::{DiscoveryNetBehaviour, config::ProtocolId}; -use crate::protocol::legacy_proto::handler::{CustomProtoHandlerProto, CustomProtoHandlerOut, CustomProtoHandlerIn}; -use crate::protocol::legacy_proto::upgrade::RegisteredProtocol; +use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; +use crate::protocol::generic_proto::handler::{NotifsHandlerProto, NotifsHandlerOut, NotifsHandlerIn}; +use crate::protocol::generic_proto::upgrade::RegisteredProtocol; + use bytes::BytesMut; +use codec::Encode as _; use fnv::FnvHashMap; use futures::prelude::*; use libp2p::core::{ConnectedPoint, Multiaddr, PeerId}; @@ -25,15 +28,32 @@ use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use log::{debug, error, trace, warn}; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; -use std::{borrow::Cow, collections::hash_map::Entry, cmp, error, marker::PhantomData, mem, pin::Pin}; -use std::time::{Duration, Instant}; +use sp_runtime::ConsensusEngineId; +use std::{borrow::Cow, collections::hash_map::Entry, cmp}; +use std::{error, mem, pin::Pin, str, time::Duration}; use std::task::{Context, Poll}; +use wasm_timer::Instant; /// Network behaviour that handles opening substreams for custom protocols with other nodes. /// +/// ## Legacy vs new protocol +/// +/// The `GenericProto` behaves as following: +/// +/// - Whenever a connection is established, we open a single substream (called "legay protocol" in +/// the source code). This substream name depends on the `protocol_id` and `versions` passed at +/// initialization. If the remote refuses this substream, we close the connection. +/// +/// - For each registered protocol, we also open an additional substream for this protocol. If the +/// remote refuses this substream, then it's fine. +/// +/// - Whenever we want to send a message, we can call either `send_packet` to force the legacy +/// substream, or `write_notification` to indicate a registered protocol. If the registered +/// protocol was refused or isn't supported by the remote, we always use the legacy instead. +/// /// ## How it works /// -/// The role of the `LegacyProto` is to synchronize the following components: +/// The role of the `GenericProto` is to synchronize the following components: /// /// - The libp2p swarm that opens new connections and reports disconnects. /// - The connection handler (see `handler.rs`) that handles individual connections. @@ -59,9 +79,12 @@ use std::task::{Context, Poll}; /// Note that this "banning" system is not an actual ban. If a "banned" node tries to connect to /// us, we accept the connection. The "banning" system is only about delaying dialing attempts. /// -pub struct LegacyProto< TSubstream> { - /// List of protocols to open with peers. Never modified. - protocol: RegisteredProtocol, +pub struct GenericProto { + /// Legacy protocol to open with peers. Never modified. + legacy_protocol: RegisteredProtocol, + + /// Notification protocols. Entries are only ever added and not removed. + notif_protocols: Vec<(Cow<'static, [u8]>, ConsensusEngineId, Vec)>, /// Receiver for instructions about who to connect to or disconnect from. peerset: sc_peerset::Peerset, @@ -78,10 +101,7 @@ pub struct LegacyProto< TSubstream> { next_incoming_index: sc_peerset::IncomingIndex, /// Events to produce from `poll()`. - events: SmallVec<[NetworkBehaviourAction; 4]>, - - /// Marker to pin the generics. - marker: PhantomData, + events: SmallVec<[NetworkBehaviourAction; 4]>, } /// State of a peer we're connected to. @@ -185,13 +205,11 @@ struct IncomingPeer { incoming_id: sc_peerset::IncomingIndex, } -/// Event that can be emitted by the `LegacyProto`. +/// Event that can be emitted by the `GenericProto`. #[derive(Debug)] -pub enum LegacyProtoOut { +pub enum GenericProtoOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { - /// Version of the protocol that has been opened. - version: u8, /// Id of the node we have opened a connection with. peer_id: PeerId, /// Endpoint used for this custom protocol. @@ -207,6 +225,8 @@ pub enum LegacyProtoOut { }, /// Receives a message on a custom protocol substream. + /// + /// Also concerns received notifications for the notifications API. CustomMessage { /// Id of the peer the message came from. peer_id: PeerId, @@ -224,26 +244,39 @@ pub enum LegacyProtoOut { }, } -impl LegacyProto { +impl GenericProto { /// Creates a `CustomProtos`. pub fn new( protocol: impl Into, versions: &[u8], peerset: sc_peerset::Peerset, ) -> Self { - let protocol = RegisteredProtocol::new(protocol, versions); + let legacy_protocol = RegisteredProtocol::new(protocol, versions); - LegacyProto { - protocol, + GenericProto { + legacy_protocol, + notif_protocols: Vec::new(), peerset, peers: FnvHashMap::default(), incoming: SmallVec::new(), next_incoming_index: sc_peerset::IncomingIndex(0), events: SmallVec::new(), - marker: PhantomData, } } + /// Registers a new notifications protocol. + /// + /// You are very strongly encouraged to call this method very early on. Any open connection + /// will retain the protocols that were registered then, and not any new one. + pub fn register_notif_protocol( + &mut self, + protocol_name: impl Into>, + engine_id: ConsensusEngineId, + handshake_msg: impl Into> + ) { + self.notif_protocols.push((protocol_name.into(), engine_id, handshake_msg.into())); + } + /// Returns the list of all the peers we have an open channel to. pub fn open_peers<'a>(&'a self) -> impl Iterator + 'a { self.peers.iter().filter(|(_, state)| state.is_open()).map(|(id, _)| id) @@ -295,7 +328,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } @@ -316,7 +349,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until } @@ -342,6 +375,44 @@ impl LegacyProto { } } + /// Sends a notification to a peer. + /// + /// Has no effect if the custom protocol is not open with the given peer. + /// + /// Also note that even if we have a valid open substream, it may in fact be already closed + /// without us knowing, in which case the packet will not be received. + /// + /// > **Note**: Ideally the `engine_id` parameter wouldn't be necessary. See the documentation + /// > of [`NotifsHandlerIn`] for more information. + pub fn write_notification( + &mut self, + target: &PeerId, + engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, + message: impl Into>, + ) { + if !self.is_open(target) { + return; + } + + trace!( + target: "sub-libp2p", + "External API => Notification for {:?} with protocol {:?}", + target, + str::from_utf8(&protocol_name) + ); + trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); + + self.events.push(NetworkBehaviourAction::SendEvent { + peer_id: target.clone(), + event: NotifsHandlerIn::SendNotification { + message: message.into(), + engine_id, + protocol_name, + }, + }); + } + /// Sends a message to a peer. /// /// Has no effect if the custom protocol is not open with the given peer. @@ -357,7 +428,7 @@ impl LegacyProto { trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: target.clone(), - event: CustomProtoHandlerIn::SendCustomMessage { + event: NotifsHandlerIn::SendLegacy { message, } }); @@ -382,12 +453,14 @@ impl LegacyProto { } }; + let now = Instant::now(); + match mem::replace(occ_entry.get_mut(), PeerState::Poisoned) { - PeerState::Banned { ref until } if *until > Instant::now() => { + PeerState::Banned { ref until } if *until > now => { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Will start to connect at \ until {:?}", occ_entry.key(), until); *occ_entry.into_mut() = PeerState::PendingRequest { - timer: futures_timer::Delay::new_at(until.clone()), + timer: futures_timer::Delay::new(until.clone() - now), timer_deadline: until.clone(), }; }, @@ -400,13 +473,13 @@ impl LegacyProto { }, PeerState::Disabled { open, ref connected_point, banned_until: Some(ref banned) } - if *banned > Instant::now() => { + if *banned > now => { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Has idle connection through \ {:?} but node is banned until {:?}", occ_entry.key(), connected_point, banned); *occ_entry.into_mut() = PeerState::DisabledPendingEnable { connected_point: connected_point.clone(), open, - timer: futures_timer::Delay::new_at(banned.clone()), + timer: futures_timer::Delay::new(banned.clone() - now), timer_deadline: banned.clone(), }; }, @@ -417,7 +490,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: occ_entry.key().clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *occ_entry.into_mut() = PeerState::Enabled { connected_point, open }; }, @@ -435,7 +508,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: occ_entry.key().clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *occ_entry.into_mut() = PeerState::Enabled { connected_point, open: false }; }, @@ -492,7 +565,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: entry.key().clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until: None } }, @@ -556,7 +629,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: incoming.peer_id, - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *state = PeerState::Enabled { open: false, connected_point }; @@ -598,13 +671,13 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: incoming.peer_id, - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *state = PeerState::Disabled { open: false, connected_point, banned_until: None }; } } -impl DiscoveryNetBehaviour for LegacyProto { +impl DiscoveryNetBehaviour for GenericProto { fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { self.peerset.discovered(peer_ids.into_iter().map(|peer_id| { debug!(target: "sub-libp2p", "PSM <= Discovered({:?})", peer_id); @@ -613,15 +686,12 @@ impl DiscoveryNetBehaviour for LegacyProto { } } -impl NetworkBehaviour for LegacyProto -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ - type ProtocolsHandler = CustomProtoHandlerProto; - type OutEvent = LegacyProtoOut; +impl NetworkBehaviour for GenericProto { + type ProtocolsHandler = NotifsHandlerProto; + type OutEvent = GenericProtoOut; fn new_handler(&mut self) -> Self::ProtocolsHandler { - CustomProtoHandlerProto::new(self.protocol.clone()) + NotifsHandlerProto::new(self.legacy_protocol.clone(), self.notif_protocols.clone()) } fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { @@ -638,7 +708,7 @@ where debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *st = PeerState::Enabled { open: false, connected_point }; } @@ -681,7 +751,7 @@ where debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *st = PeerState::Disabled { open: false, connected_point, banned_until }; } @@ -711,7 +781,7 @@ where } if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -728,7 +798,7 @@ where self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -750,7 +820,7 @@ where if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -821,10 +891,10 @@ where fn inject_node_event( &mut self, source: PeerId, - event: CustomProtoHandlerOut, + event: NotifsHandlerOut, ) { match event { - CustomProtoHandlerOut::CustomProtocolClosed { reason } => { + NotifsHandlerOut::Closed { reason } => { debug!(target: "sub-libp2p", "Handler({:?}) => Closed: {}", source, reason); let mut entry = if let Entry::Occupied(entry) = self.peers.entry(source.clone()) { @@ -835,7 +905,7 @@ where }; debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { reason, peer_id: source.clone(), }; @@ -851,7 +921,7 @@ where debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", source); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: source.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *entry.into_mut() = PeerState::Disabled { @@ -877,8 +947,8 @@ where } } - CustomProtoHandlerOut::CustomProtocolOpen { version } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Open: version {:?}", source, version); + NotifsHandlerOut::Open => { + debug!(target: "sub-libp2p", "Handler({:?}) => Open", source); let endpoint = match self.peers.get_mut(&source) { Some(PeerState::Enabled { ref mut open, ref connected_point }) | Some(PeerState::DisabledPendingEnable { ref mut open, ref connected_point, .. }) | @@ -893,8 +963,7 @@ where }; debug!(target: "sub-libp2p", "External API <= Open({:?})", source); - let event = LegacyProtoOut::CustomProtocolOpen { - version, + let event = GenericProtoOut::CustomProtocolOpen { peer_id: source, endpoint, }; @@ -902,11 +971,11 @@ where self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } - CustomProtoHandlerOut::CustomMessage { message } => { + NotifsHandlerOut::CustomMessage { message } => { debug_assert!(self.is_open(&source)); trace!(target: "sub-libp2p", "Handler({:?}) => Message", source); trace!(target: "sub-libp2p", "External API <= Message({:?})", source); - let event = LegacyProtoOut::CustomMessage { + let event = GenericProtoOut::CustomMessage { peer_id: source, message, }; @@ -914,25 +983,50 @@ where self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } - CustomProtoHandlerOut::Clogged { messages } => { + NotifsHandlerOut::Notification { protocol_name, engine_id, message } => { + debug_assert!(self.is_open(&source)); + trace!( + target: "sub-libp2p", + "Handler({:?}) => Notification({:?})", + source, + str::from_utf8(&protocol_name) + ); + trace!(target: "sub-libp2p", "External API <= Message({:?})", source); + let event = GenericProtoOut::CustomMessage { + peer_id: source, + message: { + let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { + engine_id, + data: message.to_vec(), + }); + + // Note that we clone `message` here. + From::from(&message.encode()[..]) + }, + }; + + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } + + NotifsHandlerOut::Clogged { messages } => { debug_assert!(self.is_open(&source)); trace!(target: "sub-libp2p", "Handler({:?}) => Clogged", source); trace!(target: "sub-libp2p", "External API <= Clogged({:?})", source); warn!(target: "sub-libp2p", "Queue of packets to send to {:?} is \ pretty large", source); - self.events.push(NetworkBehaviourAction::GenerateEvent(LegacyProtoOut::Clogged { + self.events.push(NetworkBehaviourAction::GenerateEvent(GenericProtoOut::Clogged { peer_id: source, messages, })); } // Don't do anything for non-severe errors except report them. - CustomProtoHandlerOut::ProtocolError { is_severe, ref error } if !is_severe => { + NotifsHandlerOut::ProtocolError { is_severe, ref error } if !is_severe => { debug!(target: "sub-libp2p", "Handler({:?}) => Benign protocol error: {:?}", source, error) } - CustomProtoHandlerOut::ProtocolError { error, .. } => { + NotifsHandlerOut::ProtocolError { error, .. } => { debug!(target: "sub-libp2p", "Handler({:?}) => Severe protocol error: {:?}", source, error); // A severe protocol error happens when we detect a "bad" node, such as a node on @@ -954,7 +1048,7 @@ where _params: &mut impl PollParameters, ) -> Poll< NetworkBehaviourAction< - CustomProtoHandlerIn, + NotifsHandlerIn, Self::OutEvent, >, > { @@ -1009,7 +1103,7 @@ where debug!(target: "sub-libp2p", "Handler({:?}) <= Enable now that ban has expired", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *peer_state = PeerState::Enabled { connected_point, open }; } diff --git a/primitives/core/src/tests.rs b/client/network/src/protocol/generic_proto/handler.rs similarity index 77% rename from primitives/core/src/tests.rs rename to client/network/src/protocol/generic_proto/handler.rs index 1eda2157c410686d37f14fae6e2af4d6b008db84..e97176cfbbfbb98cfa6f1f1d2f25dc9f5f777990 100644 --- a/primitives/core/src/tests.rs +++ b/client/network/src/protocol/generic_proto/handler.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,4 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Tests. +pub use self::group::{NotifsHandlerProto, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}; + +mod group; +mod legacy; +mod notif_in; +mod notif_out; diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs new file mode 100644 index 0000000000000000000000000000000000000000..d6d9919d3e14df0c2261731cddb0811aa291f920 --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/group.rs @@ -0,0 +1,523 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for both incoming +//! and outgoing substreams for all gossiping protocols together. +//! +//! This is the main implementation of `ProtocolsHandler` in this crate, that handles all the +//! protocols that are Substrate-related and outside of the scope of libp2p. +//! +//! # Usage +//! +//! The handler can be in one of the following states: `Initial`, `Enabled`, `Disabled`. +//! +//! The `Initial` state is the state that the handler initially is in. It is a temporary state +//! during which the user must either enable or disable the handler. After that, the handler stays +//! either enabled or disabled. +//! +//! On the wire, we try to open the following substreams: +//! +//! - One substream for each notification protocol passed as parameter to the +//! `NotifsHandlerProto::new` function. +//! - One "legacy" substream used for anything non-related to gossiping, and used as a fallback +//! in case the notification protocol can't be opened. +//! +//! When the handler is in the `Enabled` state, we immediately open and try to maintain all the +//! aforementioned substreams. When the handler is in the `Disabled` state, we immediately close +//! (or abort opening) all these substreams. It is intended that in the future we allow states in +//! which some protocols are open and not others. Symmetrically, we allow incoming +//! Substrate-related substreams if and only if we are in the `Enabled` state. +//! +//! The user has the choice between sending a message with `SendNotification`, to send a +//! notification, and `SendLegacy`, to send any other kind of message. +//! + +use crate::protocol::generic_proto::{ + handler::legacy::{LegacyProtoHandler, LegacyProtoHandlerProto, LegacyProtoHandlerIn, LegacyProtoHandlerOut}, + handler::notif_in::{NotifsInHandlerProto, NotifsInHandler, NotifsInHandlerIn, NotifsInHandlerOut}, + handler::notif_out::{NotifsOutHandlerProto, NotifsOutHandler, NotifsOutHandlerIn, NotifsOutHandlerOut}, + upgrade::{NotificationsIn, NotificationsOut, NotificationsHandshakeError, RegisteredProtocol, UpgradeCollec}, +}; +use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; + +use bytes::BytesMut; +use codec::Encode as _; +use libp2p::core::{either::{EitherError, EitherOutput}, ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{EitherUpgrade, UpgradeError, SelectUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::error; +use sp_runtime::ConsensusEngineId; +use std::{borrow::Cow, error, io, task::{Context, Poll}}; + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsHandler`]. +/// +/// See the documentation at the module level for more information. +pub struct NotifsHandlerProto { + /// Prototypes for handlers for inbound substreams. + in_handlers: Vec<(NotifsInHandlerProto, ConsensusEngineId)>, + + /// Prototypes for handlers for outbound substreams. + out_handlers: Vec<(NotifsOutHandlerProto, ConsensusEngineId)>, + + /// Prototype for handler for backwards-compatibility. + legacy: LegacyProtoHandlerProto, +} + +/// The actual handler once the connection has been established. +/// +/// See the documentation at the module level for more information. +pub struct NotifsHandler { + /// Handlers for inbound substreams. + in_handlers: Vec<(NotifsInHandler, ConsensusEngineId)>, + + /// Handlers for outbound substreams. + out_handlers: Vec<(NotifsOutHandler, ConsensusEngineId)>, + + /// Handler for backwards-compatibility. + legacy: LegacyProtoHandler, + + /// State of this handler. + enabled: EnabledState, + + /// If we receive inbound substream requests while in initialization mode, + /// we push the corresponding index here and process them when the handler + /// gets enabled/disabled. + pending_in: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum EnabledState { + Initial, + Enabled, + Disabled, +} + +impl IntoProtocolsHandler for NotifsHandlerProto { + type Handler = NotifsHandler; + + fn inbound_protocol(&self) -> SelectUpgrade, RegisteredProtocol> { + let in_handlers = self.in_handlers.iter() + .map(|(h, _)| h.inbound_protocol()) + .collect::>(); + + SelectUpgrade::new(in_handlers, self.legacy.inbound_protocol()) + } + + fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { + NotifsHandler { + in_handlers: self.in_handlers + .into_iter() + .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .collect(), + out_handlers: self.out_handlers + .into_iter() + .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .collect(), + legacy: self.legacy.into_handler(remote_peer_id, connected_point), + enabled: EnabledState::Initial, + pending_in: Vec::new(), + } + } +} + +/// Event that can be received by a `NotifsHandler`. +#[derive(Debug)] +pub enum NotifsHandlerIn { + /// The node should start using custom protocols. + Enable, + + /// The node should stop using custom protocols. + Disable, + + /// Sends a message through the custom protocol substream. + /// + /// > **Note**: This must **not** be an encoded `ConsensusMessage` message. + SendLegacy { + /// The message to send. + message: Vec, + }, + + /// Sends a notifications message. + SendNotification { + /// Name of the protocol for the message. + /// + /// Must match one of the registered protocols. For backwards-compatibility reasons, if + /// the remote doesn't support this protocol, we use the legacy substream to send a + /// `ConsensusMessage` message. + protocol_name: Cow<'static, [u8]>, + + /// The engine ID to use, in case we need to send this message over the legacy substream. + /// + /// > **Note**: Ideally this field wouldn't be necessary, and we would deduce the engine + /// > ID from the existing handlers. However, it is possible (especially in test + /// > situations) that we open connections before all the notification protocols + /// > have been registered, in which case we always rely on the legacy substream. + engine_id: ConsensusEngineId, + + /// The message to send. + message: Vec, + }, +} + +/// Event that can be emitted by a `NotifsHandler`. +#[derive(Debug)] +pub enum NotifsHandlerOut { + /// Opened the substreams with the remote. + Open, + + /// Closed the substreams with the remote. + Closed { + /// Reason why the substream closed, for diagnostic purposes. + reason: Cow<'static, str>, + }, + + /// Received a non-gossiping message on the legacy substream. + CustomMessage { + /// Message that has been received. + /// + /// Keep in mind that this can be a `ConsensusMessage` message, which then contains a + /// notification. + message: BytesMut, + }, + + /// Received a message on a custom protocol substream. + Notification { + /// Engine corresponding to the message. + protocol_name: Cow<'static, [u8]>, + + /// For legacy reasons, the name to use if we had received the message from the legacy + /// substream. + engine_id: ConsensusEngineId, + + /// Message that has been received. + /// + /// If `protocol_name` is `None`, this decodes to a `Message`. If `protocol_name` is `Some`, + /// this is directly a gossiping message. + message: BytesMut, + }, + + /// A substream to the remote is clogged. The send buffer is very large, and we should print + /// a diagnostic message and/or avoid sending more data. + Clogged { + /// Copy of the messages that are within the buffer, for further diagnostic. + messages: Vec>, + }, + + /// An error has happened on the protocol level with this node. + ProtocolError { + /// If true the error is severe, such as a protocol violation. + is_severe: bool, + /// The error that happened. + error: Box, + }, +} + +impl NotifsHandlerProto { + /// Builds a new handler. + pub fn new(legacy: RegisteredProtocol, list: impl Into, ConsensusEngineId, Vec)>>) -> Self { + let list = list.into(); + + NotifsHandlerProto { + in_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsInHandlerProto::new(p), e)).collect(), + out_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsOutHandlerProto::new(p), e)).collect(), + legacy: LegacyProtoHandlerProto::new(legacy), + } + } +} + +impl ProtocolsHandler for NotifsHandler { + type InEvent = NotifsHandlerIn; + type OutEvent = NotifsHandlerOut; + type Error = EitherError< + EitherError< + ::Error, + ::Error, + >, + ::Error, + >; + type InboundProtocol = SelectUpgrade, RegisteredProtocol>; + type OutboundProtocol = EitherUpgrade; + // Index within the `out_handlers`; None for legacy + type OutboundOpenInfo = Option; + + fn listen_protocol(&self) -> SubstreamProtocol { + let in_handlers = self.in_handlers.iter() + .map(|h| h.0.listen_protocol().into_upgrade().1) + .collect::>(); + + let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1); + SubstreamProtocol::new(proto) + } + + fn inject_fully_negotiated_inbound( + &mut self, + out: >::Output + ) { + match out { + EitherOutput::First((out, num)) => + self.in_handlers[num].0.inject_fully_negotiated_inbound(out), + EitherOutput::Second(out) => + self.legacy.inject_fully_negotiated_inbound(out), + } + } + + fn inject_fully_negotiated_outbound( + &mut self, + out: >::Output, + num: Self::OutboundOpenInfo + ) { + match (out, num) { + (EitherOutput::First(out), Some(num)) => + self.out_handlers[num].0.inject_fully_negotiated_outbound(out, ()), + (EitherOutput::Second(out), None) => + self.legacy.inject_fully_negotiated_outbound(out, ()), + _ => error!("inject_fully_negotiated_outbound called with wrong parameters"), + } + } + + fn inject_event(&mut self, message: NotifsHandlerIn) { + match message { + NotifsHandlerIn::Enable => { + self.enabled = EnabledState::Enabled; + self.legacy.inject_event(LegacyProtoHandlerIn::Enable); + for (handler, _) in &mut self.out_handlers { + handler.inject_event(NotifsOutHandlerIn::Enable { + initial_message: vec![] + }); + } + for num in self.pending_in.drain(..) { + self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Accept(vec![])); + } + }, + NotifsHandlerIn::Disable => { + self.legacy.inject_event(LegacyProtoHandlerIn::Disable); + // The notifications protocols start in the disabled state. If we were in the + // "Initial" state, then we shouldn't disable the notifications protocols again. + if self.enabled != EnabledState::Initial { + for (handler, _) in &mut self.out_handlers { + handler.inject_event(NotifsOutHandlerIn::Disable); + } + } + self.enabled = EnabledState::Disabled; + for num in self.pending_in.drain(..) { + self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Refuse); + } + }, + NotifsHandlerIn::SendLegacy { message } => + self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { message }), + NotifsHandlerIn::SendNotification { message, engine_id, protocol_name } => { + for (handler, ngn_id) in &mut self.out_handlers { + if handler.protocol_name() != &protocol_name[..] { + break; + } + + if handler.is_open() { + handler.inject_event(NotifsOutHandlerIn::Send(message)); + return; + } else { + debug_assert_eq!(engine_id, *ngn_id); + } + } + + let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { + engine_id, + data: message, + }); + + self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { + message: message.encode() + }); + }, + } + } + + fn inject_dial_upgrade_error( + &mut self, + num: Option, + err: ProtocolsHandlerUpgrErr> + ) { + match (err, num) { + (ProtocolsHandlerUpgrErr::Timeout, Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Timeout + ), + (ProtocolsHandlerUpgrErr::Timeout, None) => + self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timeout), + (ProtocolsHandlerUpgrErr::Timer, Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Timer + ), + (ProtocolsHandlerUpgrErr::Timer, None) => + self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timer), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), None) => + self.legacy.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::A(err))), Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::B(err))), None) => + self.legacy.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) + ), + _ => error!("inject_dial_upgrade_error called with bad parameters"), + } + } + + fn connection_keep_alive(&self) -> KeepAlive { + // Iterate over each handler and return the maximum value. + + let mut ret = self.legacy.connection_keep_alive(); + if ret.is_yes() { + return KeepAlive::Yes; + } + + for (handler, _) in &self.in_handlers { + let val = handler.connection_keep_alive(); + if val.is_yes() { + return KeepAlive::Yes; + } + if ret < val { ret = val; } + } + + for (handler, _) in &self.out_handlers { + let val = handler.connection_keep_alive(); + if val.is_yes() { + return KeepAlive::Yes; + } + if ret < val { ret = val; } + } + + ret + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll< + ProtocolsHandlerEvent + > { + for (handler_num, (handler, engine_id)) in self.in_handlers.iter_mut().enumerate() { + while let Poll::Ready(ev) = handler.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } => + error!("Incoming substream handler tried to open a substream"), + ProtocolsHandlerEvent::Close(err) => void::unreachable(err), + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(_)) => + match self.enabled { + EnabledState::Initial => self.pending_in.push(handler_num), + EnabledState::Enabled => + handler.inject_event(NotifsInHandlerIn::Accept(vec![])), + EnabledState::Disabled => + handler.inject_event(NotifsInHandlerIn::Refuse), + }, + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {}, + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => { + // Note that right now the legacy substream has precedence over + // everything. If it is not open, then we consider that nothing is open. + if self.legacy.is_open() { + let msg = NotifsHandlerOut::Notification { + message, + engine_id: *engine_id, + protocol_name: handler.protocol_name().to_owned().into(), + }; + return Poll::Ready(ProtocolsHandlerEvent::Custom(msg)); + } + }, + } + } + } + + for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() { + while let Poll::Ready(ev) = handler.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => + return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: protocol.map_upgrade(EitherUpgrade::A), + info: Some(handler_num), + }), + ProtocolsHandlerEvent::Close(err) => void::unreachable(err), + + // At the moment we don't actually care whether any notifications protocol + // opens or closes. + // Whether our communications with the remote are open or closed entirely + // depends on the legacy substream, because as long as we are open the user of + // this struct might try to send legacy protocol messages which we need to + // deliver for things to work properly. + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Open { .. }) => {}, + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed) => {}, + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Refused) => {}, + } + } + } + + while let Poll::Ready(ev) = self.legacy.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => + return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: protocol.map_upgrade(EitherUpgrade::B), + info: None, + }), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { .. }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Open + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Closed { reason } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::CustomMessage { message } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::Clogged { messages }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Clogged { messages } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::ProtocolError { is_severe, error } + )), + ProtocolsHandlerEvent::Close(err) => + return Poll::Ready(ProtocolsHandlerEvent::Close(EitherError::B(err))), + } + } + + Poll::Pending + } +} diff --git a/client/network/src/protocol/legacy_proto/handler.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs similarity index 84% rename from client/network/src/protocol/legacy_proto/handler.rs rename to client/network/src/protocol/generic_proto/handler/legacy.rs index fc3b64b968c010c6f2253a05a2c1a81a7c486d5d..a2d2fc9246d1c79b761d5e5eb1ce375fba33b49a 100644 --- a/client/network/src/protocol/legacy_proto/handler.rs +++ b/client/network/src/protocol/generic_proto/handler/legacy.rs @@ -14,11 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use super::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream}; +use crate::protocol::generic_proto::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream}; use bytes::BytesMut; use futures::prelude::*; use futures_timer::Delay; -use libp2p::core::{ConnectedPoint, Negotiated, PeerId, Endpoint}; +use libp2p::core::{ConnectedPoint, PeerId, Endpoint}; use libp2p::core::upgrade::{InboundUpgrade, OutboundUpgrade}; use libp2p::swarm::{ ProtocolsHandler, ProtocolsHandlerEvent, @@ -26,17 +26,18 @@ use libp2p::swarm::{ KeepAlive, ProtocolsHandlerUpgrErr, SubstreamProtocol, + NegotiatedSubstream, }; use log::{debug, error}; use smallvec::{smallvec, SmallVec}; -use std::{borrow::Cow, error, fmt, io, marker::PhantomData, mem, time::Duration}; +use std::{borrow::Cow, error, fmt, io, mem, time::Duration}; use std::{pin::Pin, task::{Context, Poll}}; /// Implements the `IntoProtocolsHandler` trait of libp2p. /// /// Every time a connection with a remote starts, an instance of this struct is created and /// sent to a background task dedicated to this connection. Once the connection is established, -/// it is turned into a `CustomProtoHandler`. It then handles all communications that are specific +/// it is turned into a `LegacyProtoHandler`. It then handles all communications that are specific /// to Substrate on that single connection. /// /// Note that there can be multiple instance of this struct simultaneously for same peer. However @@ -86,39 +87,29 @@ use std::{pin::Pin, task::{Context, Poll}}; /// We consider that we are now "closed" if the remote closes all the existing substreams. /// Re-opening it can then be performed by closing all active substream and re-opening one. /// -pub struct CustomProtoHandlerProto { +pub struct LegacyProtoHandlerProto { /// Configuration for the protocol upgrade to negotiate. protocol: RegisteredProtocol, - - /// Marker to pin the generic type. - marker: PhantomData, } -impl CustomProtoHandlerProto -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ - /// Builds a new `CustomProtoHandlerProto`. +impl LegacyProtoHandlerProto { + /// Builds a new `LegacyProtoHandlerProto`. pub fn new(protocol: RegisteredProtocol) -> Self { - CustomProtoHandlerProto { + LegacyProtoHandlerProto { protocol, - marker: PhantomData, } } } -impl IntoProtocolsHandler for CustomProtoHandlerProto -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ - type Handler = CustomProtoHandler; +impl IntoProtocolsHandler for LegacyProtoHandlerProto { + type Handler = LegacyProtoHandler; fn inbound_protocol(&self) -> RegisteredProtocol { self.protocol.clone() } fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { - CustomProtoHandler { + LegacyProtoHandler { protocol: self.protocol, endpoint: connected_point.to_endpoint(), remote_peer_id: remote_peer_id.clone(), @@ -132,12 +123,12 @@ where } /// The actual handler once the connection has been established. -pub struct CustomProtoHandler { +pub struct LegacyProtoHandler { /// Configuration for the protocol upgrade to negotiate. protocol: RegisteredProtocol, /// State of the communications with the remote. - state: ProtocolState, + state: ProtocolState, /// Identifier of the node we're talking to. Used only for logging purposes and shouldn't have /// any influence on the behaviour. @@ -151,15 +142,15 @@ pub struct CustomProtoHandler { /// /// This queue must only ever be modified to insert elements at the back, or remove the first /// element. - events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, } /// State of the handler. -enum ProtocolState { +enum ProtocolState { /// Waiting for the behaviour to tell the handler whether it is enabled or disabled. Init { /// List of substreams opened by the remote but that haven't been processed yet. - substreams: SmallVec<[RegisteredProtocolSubstream>; 6]>, + substreams: SmallVec<[RegisteredProtocolSubstream; 6]>, /// Deadline after which the initialization is abnormally long. init_deadline: Delay, }, @@ -175,9 +166,9 @@ enum ProtocolState { /// If we are in this state, we have sent a `CustomProtocolOpen` message to the outside. Normal { /// The substreams where bidirectional communications happen. - substreams: SmallVec<[RegisteredProtocolSubstream>; 4]>, + substreams: SmallVec<[RegisteredProtocolSubstream; 4]>, /// Contains substreams which are being shut down. - shutdown: SmallVec<[RegisteredProtocolSubstream>; 4]>, + shutdown: SmallVec<[RegisteredProtocolSubstream; 4]>, }, /// We are disabled. Contains substreams that are being closed. @@ -185,7 +176,7 @@ enum ProtocolState { /// outside or we have never sent any `CustomProtocolOpen` in the first place. Disabled { /// List of substreams to shut down. - shutdown: SmallVec<[RegisteredProtocolSubstream>; 6]>, + shutdown: SmallVec<[RegisteredProtocolSubstream; 6]>, /// If true, we should reactivate the handler after all the substreams in `shutdown` have /// been closed. @@ -204,9 +195,9 @@ enum ProtocolState { Poisoned, } -/// Event that can be received by a `CustomProtoHandler`. +/// Event that can be received by a `LegacyProtoHandler`. #[derive(Debug)] -pub enum CustomProtoHandlerIn { +pub enum LegacyProtoHandlerIn { /// The node should start using custom protocols. Enable, @@ -220,9 +211,9 @@ pub enum CustomProtoHandlerIn { }, } -/// Event that can be emitted by a `CustomProtoHandler`. +/// Event that can be emitted by a `LegacyProtoHandler`. #[derive(Debug)] -pub enum CustomProtoHandlerOut { +pub enum LegacyProtoHandlerOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { /// Version of the protocol that has been opened. @@ -257,10 +248,19 @@ pub enum CustomProtoHandlerOut { }, } -impl CustomProtoHandler -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ +impl LegacyProtoHandler { + /// Returns true if the legacy substream is currently open. + pub fn is_open(&self) -> bool { + match &self.state { + ProtocolState::Init { substreams, .. } => !substreams.is_empty(), + ProtocolState::Opening { .. } => false, + ProtocolState::Normal { substreams, .. } => !substreams.is_empty(), + ProtocolState::Disabled { .. } => false, + ProtocolState::KillAsap => false, + ProtocolState::Poisoned => false, + } + } + /// Enables the handler. fn enable(&mut self) { self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) { @@ -283,7 +283,7 @@ where } } else { - let event = CustomProtoHandlerOut::CustomProtocolOpen { + let event = LegacyProtoHandlerOut::CustomProtocolOpen { version: incoming[0].protocol_version() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); @@ -337,7 +337,7 @@ where /// Polls the state for events. Optionally returns an event to produce. #[must_use] fn poll_state(&mut self, cx: &mut Context) - -> Option> { + -> Option> { match mem::replace(&mut self.state, ProtocolState::Poisoned) { ProtocolState::Poisoned => { error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state", @@ -348,13 +348,12 @@ where ProtocolState::Init { substreams, mut init_deadline } => { match Pin::new(&mut init_deadline).poll(cx) { - Poll::Ready(Ok(())) => { + Poll::Ready(()) => { init_deadline = Delay::new(Duration::from_secs(60)); error!(target: "sub-libp2p", "Handler initialization process is too long \ with {:?}", self.remote_peer_id) }, Poll::Pending => {} - Poll::Ready(Err(_)) => error!(target: "sub-libp2p", "Tokio timer has errored") } self.state = ProtocolState::Init { substreams, init_deadline }; @@ -363,9 +362,9 @@ where ProtocolState::Opening { mut deadline } => { match Pin::new(&mut deadline).poll(cx) { - Poll::Ready(Ok(())) => { + Poll::Ready(()) => { deadline = Delay::new(Duration::from_secs(60)); - let event = CustomProtoHandlerOut::ProtocolError { + let event = LegacyProtoHandlerOut::ProtocolError { is_severe: true, error: "Timeout when opening protocol".to_string().into(), }; @@ -376,12 +375,6 @@ where self.state = ProtocolState::Opening { deadline }; None }, - Poll::Ready(Err(_)) => { - error!(target: "sub-libp2p", "Tokio timer has errored"); - deadline = Delay::new(Duration::from_secs(60)); - self.state = ProtocolState::Opening { deadline }; - None - }, } } @@ -391,7 +384,7 @@ where match Pin::new(&mut substream).poll_next(cx) { Poll::Pending => substreams.push(substream), Poll::Ready(Some(Ok(RegisteredProtocolEvent::Message(message)))) => { - let event = CustomProtoHandlerOut::CustomMessage { + let event = LegacyProtoHandlerOut::CustomMessage { message }; substreams.push(substream); @@ -399,7 +392,7 @@ where return Some(ProtocolsHandlerEvent::Custom(event)); }, Poll::Ready(Some(Ok(RegisteredProtocolEvent::Clogged { messages }))) => { - let event = CustomProtoHandlerOut::Clogged { + let event = LegacyProtoHandlerOut::Clogged { messages, }; substreams.push(substream); @@ -409,7 +402,7 @@ where Poll::Ready(None) => { shutdown.push(substream); if substreams.is_empty() { - let event = CustomProtoHandlerOut::CustomProtocolClosed { + let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: "All substreams have been closed by the remote".into(), }; self.state = ProtocolState::Disabled { @@ -421,7 +414,7 @@ where } Poll::Ready(Some(Err(err))) => { if substreams.is_empty() { - let event = CustomProtoHandlerOut::CustomProtocolClosed { + let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: format!("Error on the last substream: {:?}", err).into(), }; self.state = ProtocolState::Disabled { @@ -466,7 +459,7 @@ where /// Called by `inject_fully_negotiated_inbound` and `inject_fully_negotiated_outbound`. fn inject_fully_negotiated( &mut self, - mut substream: RegisteredProtocolSubstream> + mut substream: RegisteredProtocolSubstream ) { self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) { ProtocolState::Poisoned => { @@ -485,7 +478,7 @@ where } ProtocolState::Opening { .. } => { - let event = CustomProtoHandlerOut::CustomProtocolOpen { + let event = LegacyProtoHandlerOut::CustomProtocolOpen { version: substream.protocol_version() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); @@ -522,11 +515,9 @@ where } } -impl ProtocolsHandler for CustomProtoHandler -where TSubstream: AsyncRead + AsyncWrite + Unpin { - type InEvent = CustomProtoHandlerIn; - type OutEvent = CustomProtoHandlerOut; - type Substream = TSubstream; +impl ProtocolsHandler for LegacyProtoHandler { + type InEvent = LegacyProtoHandlerIn; + type OutEvent = LegacyProtoHandlerOut; type Error = ConnectionKillError; type InboundProtocol = RegisteredProtocol; type OutboundProtocol = RegisteredProtocol; @@ -538,24 +529,24 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin { fn inject_fully_negotiated_inbound( &mut self, - proto: >>::Output + proto: >::Output ) { self.inject_fully_negotiated(proto); } fn inject_fully_negotiated_outbound( &mut self, - proto: >>::Output, + proto: >::Output, _: Self::OutboundOpenInfo ) { self.inject_fully_negotiated(proto); } - fn inject_event(&mut self, message: CustomProtoHandlerIn) { + fn inject_event(&mut self, message: LegacyProtoHandlerIn) { match message { - CustomProtoHandlerIn::Disable => self.disable(), - CustomProtoHandlerIn::Enable => self.enable(), - CustomProtoHandlerIn::SendCustomMessage { message } => + LegacyProtoHandlerIn::Disable => self.disable(), + LegacyProtoHandlerIn::Enable => self.enable(), + LegacyProtoHandlerIn::SendCustomMessage { message } => self.send_message(message), } } @@ -567,7 +558,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin { _ => false, }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(CustomProtoHandlerOut::ProtocolError { + self.events_queue.push(ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error: Box::new(err), })); @@ -608,22 +599,19 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin { } } -impl fmt::Debug for CustomProtoHandler -where - TSubstream: AsyncRead + AsyncWrite + Unpin, -{ +impl fmt::Debug for LegacyProtoHandler { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - f.debug_struct("CustomProtoHandler") + f.debug_struct("LegacyProtoHandler") .finish() } } /// Given a list of substreams, tries to shut them down. The substreams that have been successfully /// shut down are removed from the list. -fn shutdown_list - (list: &mut SmallVec>>>, +fn shutdown_list + (list: &mut SmallVec>>, cx: &mut Context) -where TSubstream: AsyncRead + AsyncWrite + Unpin { +{ 'outer: for n in (0..list.len()).rev() { let mut substream = list.swap_remove(n); loop { diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs new file mode 100644 index 0000000000000000000000000000000000000000..4e16fb1af419f4aebb2748b09c477119f1165309 --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -0,0 +1,256 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for ingoing +//! substreams for a single gossiping protocol. +//! +//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple +//! > protocols, you need to create multiple instances and group them. +//! + +use crate::protocol::generic_proto::upgrade::{NotificationsIn, NotificationsInSubstream}; +use bytes::BytesMut; +use futures::prelude::*; +use libp2p::core::{ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::{error, warn}; +use smallvec::SmallVec; +use std::{borrow::Cow, fmt, pin::Pin, str, task::{Context, Poll}}; + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsInHandler`]. +pub struct NotifsInHandlerProto { + /// Configuration for the protocol upgrade to negotiate. + in_protocol: NotificationsIn, +} + +/// The actual handler once the connection has been established. +pub struct NotifsInHandler { + /// Configuration for the protocol upgrade to negotiate for inbound substreams. + in_protocol: NotificationsIn, + + /// Substream that is open with the remote. + substream: Option>, + + /// If the substream is opened and closed rapidly, we can emit several `OpenRequest` and + /// `Closed` messages in a row without the handler having time to respond with `Accept` or + /// `Refuse`. + /// + /// In order to keep the state consistent, we increment this variable every time an + /// `OpenRequest` is emitted and decrement it every time an `Accept` or `Refuse` is received. + pending_accept_refuses: usize, + + /// Queue of events to send to the outside. + /// + /// This queue is only ever modified to insert elements at the back, or remove the first + /// element. + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, +} + +/// Event that can be received by a `NotifsInHandler`. +#[derive(Debug)] +pub enum NotifsInHandlerIn { + /// Can be sent back as a response to an `OpenRequest`. Contains the status message to send + /// to the remote. + /// + /// After sending this to the handler, the substream is now considered open and `Notif` events + /// can be received. + Accept(Vec), + + /// Can be sent back as a response to an `OpenRequest`. + Refuse, +} + +/// Event that can be emitted by a `NotifsInHandler`. +#[derive(Debug)] +pub enum NotifsInHandlerOut { + /// The remote wants to open a substream. Contains the initial message sent by the remote + /// when the substream has been opened. + /// + /// Every time this event is emitted, a corresponding `Accepted` or `Refused` **must** be sent + /// back even if a `Closed` is received. + OpenRequest(Vec), + + /// The notifications substream has been closed by the remote. In order to avoid race + /// conditions, this does **not** cancel any previously-sent `OpenRequest`. + Closed, + + /// Received a message on the notifications substream. + /// + /// Can only happen after an `Accept` and before a `Closed`. + Notif(BytesMut), +} + +impl NotifsInHandlerProto { + /// Builds a new `NotifsInHandlerProto`. + pub fn new( + protocol_name: impl Into> + ) -> Self { + NotifsInHandlerProto { + in_protocol: NotificationsIn::new(protocol_name), + } + } +} + +impl IntoProtocolsHandler for NotifsInHandlerProto { + type Handler = NotifsInHandler; + + fn inbound_protocol(&self) -> NotificationsIn { + self.in_protocol.clone() + } + + fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler { + NotifsInHandler { + in_protocol: self.in_protocol, + substream: None, + pending_accept_refuses: 0, + events_queue: SmallVec::new(), + } + } +} + +impl NotifsInHandler { + /// Returns the name of the protocol that we accept. + pub fn protocol_name(&self) -> &[u8] { + self.in_protocol.protocol_name() + } +} + +impl ProtocolsHandler for NotifsInHandler { + type InEvent = NotifsInHandlerIn; + type OutEvent = NotifsInHandlerOut; + type Error = void::Void; + type InboundProtocol = NotificationsIn; + type OutboundProtocol = DeniedUpgrade; + type OutboundOpenInfo = (); + + fn listen_protocol(&self) -> SubstreamProtocol { + SubstreamProtocol::new(self.in_protocol.clone()) + } + + fn inject_fully_negotiated_inbound( + &mut self, + (msg, proto): >::Output + ) { + if self.substream.is_some() { + warn!( + target: "sub-libp2p", + "Received duplicate inbound notifications substream for {:?}", + str::from_utf8(self.in_protocol.protocol_name()), + ); + return; + } + + self.substream = Some(proto); + self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); + self.pending_accept_refuses = self.pending_accept_refuses + .checked_add(1) + .unwrap_or_else(|| { + error!(target: "sub-libp2p", "Overflow in pending_accept_refuses"); + usize::max_value() + }); + } + + fn inject_fully_negotiated_outbound( + &mut self, + out: >::Output, + _: Self::OutboundOpenInfo + ) { + // We never emit any outgoing substream. + void::unreachable(out) + } + + fn inject_event(&mut self, message: NotifsInHandlerIn) { + self.pending_accept_refuses = match self.pending_accept_refuses.checked_sub(1) { + Some(v) => v, + None => { + error!( + target: "sub-libp2p", + "Inconsistent state: received Accept/Refuse when no pending request exists" + ); + return; + } + }; + + // If we send multiple `OpenRequest`s in a row, we will receive back multiple + // `Accept`/`Refuse` messages. All of them are obsolete except the last one. + if self.pending_accept_refuses != 0 { + return; + } + + match (message, self.substream.as_mut()) { + (NotifsInHandlerIn::Accept(message), Some(sub)) => sub.send_handshake(message), + (NotifsInHandlerIn::Accept(_), None) => {}, + (NotifsInHandlerIn::Refuse, _) => self.substream = None, + } + } + + fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr) { + error!(target: "sub-libp2p", "Received dial upgrade error in inbound-only handler"); + } + + fn connection_keep_alive(&self) -> KeepAlive { + if self.substream.is_some() { + KeepAlive::Yes + } else { + KeepAlive::No + } + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll< + ProtocolsHandlerEvent + > { + // Flush the events queue if necessary. + if !self.events_queue.is_empty() { + let event = self.events_queue.remove(0); + return Poll::Ready(event) + } + + match self.substream.as_mut().map(|s| Stream::poll_next(Pin::new(s), cx)) { + None | Some(Poll::Pending) => {}, + Some(Poll::Ready(Some(Ok(msg)))) => + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(msg))), + Some(Poll::Ready(None)) | Some(Poll::Ready(Some(Err(_)))) => { + self.substream = None; + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); + }, + } + + Poll::Pending + } +} + +impl fmt::Debug for NotifsInHandler { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.debug_struct("NotifsInHandler") + .field("substream_open", &self.substream.is_some()) + .finish() + } +} diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs new file mode 100644 index 0000000000000000000000000000000000000000..8c64491d997171df73606c2852765c10c9f3f21b --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs @@ -0,0 +1,395 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for outgoing +//! substreams of a single gossiping protocol. +//! +//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple +//! > protocols, you need to create multiple instances and group them. +//! + +use crate::protocol::generic_proto::upgrade::{NotificationsOut, NotificationsOutSubstream, NotificationsHandshakeError}; +use futures::prelude::*; +use libp2p::core::{ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::error; +use smallvec::SmallVec; +use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; +use wasm_timer::Instant; + +/// Maximum duration to open a substream and receive the handshake message. After that, we +/// consider that we failed to open the substream. +const OPEN_TIMEOUT: Duration = Duration::from_secs(10); +/// After successfully establishing a connection with the remote, we keep the connection open for +/// at least this amount of time in order to give the rest of the code the chance to notify us to +/// open substreams. +const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5); + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsOutHandler`]. +/// +/// See the documentation of [`NotifsOutHandler`] for more information. +pub struct NotifsOutHandlerProto { + /// Name of the protocol to negotiate. + protocol_name: Cow<'static, [u8]>, +} + +impl NotifsOutHandlerProto { + /// Builds a new [`NotifsOutHandlerProto`]. Will use the given protocol name for the + /// notifications substream. + pub fn new(protocol_name: impl Into>) -> Self { + NotifsOutHandlerProto { + protocol_name: protocol_name.into(), + } + } +} + +impl IntoProtocolsHandler for NotifsOutHandlerProto { + type Handler = NotifsOutHandler; + + fn inbound_protocol(&self) -> DeniedUpgrade { + DeniedUpgrade + } + + fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler { + NotifsOutHandler { + protocol_name: self.protocol_name, + when_connection_open: Instant::now(), + state: State::Disabled, + events_queue: SmallVec::new(), + } + } +} + +/// Handler for an outbound notification substream. +/// +/// When a connection is established, this handler starts in the "disabled" state, meaning that +/// no substream will be open. +/// +/// One can try open a substream by sending an [`NotifsOutHandlerIn::Enable`] message to the +/// handler. Once done, the handler will try to establish then maintain an outbound substream with +/// the remote for the purpose of sending notifications to it. +pub struct NotifsOutHandler { + /// Name of the protocol to negotiate. + protocol_name: Cow<'static, [u8]>, + + /// Relationship with the node we're connected to. + state: State, + + /// When the connection with the remote has been successfully established. + when_connection_open: Instant, + + /// Queue of events to send to the outside. + /// + /// This queue must only ever be modified to insert elements at the back, or remove the first + /// element. + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, +} + +/// Our relationship with the node we're connected to. +enum State { + /// The handler is disabled and idle. No substream is open. + Disabled, + + /// The handler is disabled. A substream is still open and needs to be closed. + /// + /// > **Important**: Having this state means that `poll_close` has been called at least once, + /// > but the `Sink` API is unclear about whether or not the stream can then + /// > be recovered. Because of that, we must never switch from the + /// > `DisabledOpen` state to the `Open` state while keeping the same substream. + DisabledOpen(NotificationsOutSubstream), + + /// The handler is disabled but we are still trying to open a substream with the remote. + /// + /// If the handler gets enabled again, we can immediately switch to `Opening`. + DisabledOpening, + + /// The handler is enabled and we are trying to open a substream with the remote. + Opening { + /// The initial message that we sent. Necessary if we need to re-open a substream. + initial_message: Vec, + }, + + /// The handler is enabled. We have tried opening a substream in the past but the remote + /// refused it. + Refused, + + /// The handler is enabled and substream is open. + Open { + /// Substream that is currently open. + substream: NotificationsOutSubstream, + /// The initial message that we sent. Necessary if we need to re-open a substream. + initial_message: Vec, + }, + + /// Poisoned state. Shouldn't be found in the wild. + Poisoned, +} + +/// Event that can be received by a `NotifsOutHandler`. +#[derive(Debug)] +pub enum NotifsOutHandlerIn { + /// Enables the notifications substream for this node. The handler will try to maintain a + /// substream with the remote. + Enable { + /// Initial message to send to remote nodes when we open substreams. + initial_message: Vec, + }, + + /// Disables the notifications substream for this node. This is the default state. + Disable, + + /// Sends a message on the notifications substream. Ignored if the substream isn't open. + /// + /// It is only valid to send this if the notifications substream has been enabled. + Send(Vec), +} + +/// Event that can be emitted by a `NotifsOutHandler`. +#[derive(Debug)] +pub enum NotifsOutHandlerOut { + /// The notifications substream has been accepted by the remote. + Open { + /// Handshake message sent by the remote after we opened the substream. + handshake: Vec, + }, + + /// The notifications substream has been closed by the remote. + Closed, + + /// We tried to open a notifications substream, but the remote refused it. + /// + /// Can only happen if we're in a closed state. + Refused, +} + +impl NotifsOutHandler { + /// Returns true if the substream is currently open. + pub fn is_open(&self) -> bool { + match &self.state { + State::Disabled => false, + State::DisabledOpening => false, + State::DisabledOpen(_) => true, + State::Opening { .. } => false, + State::Refused => false, + State::Open { .. } => true, + State::Poisoned => false, + } + } + + /// Returns the name of the protocol that we negotiate. + pub fn protocol_name(&self) -> &[u8] { + &self.protocol_name + } +} + +impl ProtocolsHandler for NotifsOutHandler { + type InEvent = NotifsOutHandlerIn; + type OutEvent = NotifsOutHandlerOut; + type Error = void::Void; + type InboundProtocol = DeniedUpgrade; + type OutboundProtocol = NotificationsOut; + type OutboundOpenInfo = (); + + fn listen_protocol(&self) -> SubstreamProtocol { + SubstreamProtocol::new(DeniedUpgrade) + } + + fn inject_fully_negotiated_inbound( + &mut self, + proto: >::Output + ) { + // We should never reach here. `proto` is a `Void`. + void::unreachable(proto) + } + + fn inject_fully_negotiated_outbound( + &mut self, + (handshake_msg, substream): >::Output, + _: () + ) { + match mem::replace(&mut self.state, State::Poisoned) { + State::Opening { initial_message } => { + let ev = NotifsOutHandlerOut::Open { handshake: handshake_msg }; + self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + self.state = State::Open { substream, initial_message }; + }, + // If the handler was disabled while we were negotiating the protocol, immediately + // close it. + State::DisabledOpening => self.state = State::DisabledOpen(substream), + + // Any other situation should never happen. + State::Disabled | State::Refused | State::Open { .. } | State::DisabledOpen(_) => + error!("State mismatch in notifications handler: substream already open"), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + fn inject_event(&mut self, message: NotifsOutHandlerIn) { + match message { + NotifsOutHandlerIn::Enable { initial_message } => { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled => { + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + self.state = State::Opening { initial_message }; + }, + State::DisabledOpening => self.state = State::Opening { initial_message }, + State::DisabledOpen(mut sub) => { + // As documented above, in this state we have already called `poll_close` + // once on the substream, and it is unclear whether the substream can then + // be recovered. When in doubt, let's drop the existing substream and + // open a new one. + if sub.close().now_or_never().is_none() { + log::warn!( + target: "sub-libp2p", + "Improperly closed outbound notifications substream" + ); + } + + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + self.state = State::Opening { initial_message }; + }, + State::Opening { .. } | State::Refused | State::Open { .. } => + error!("Tried to enable notifications handler that was already enabled"), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + NotifsOutHandlerIn::Disable => { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => + error!("Tried to disable notifications handler that was already disabled"), + State::Opening { .. } => self.state = State::DisabledOpening, + State::Refused => self.state = State::Disabled, + State::Open { substream, .. } => self.state = State::DisabledOpen(substream), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + NotifsOutHandlerIn::Send(msg) => + if let State::Open { substream, .. } = &mut self.state { + if let Some(Ok(_)) = substream.send(msg).now_or_never() { + } else { + log::warn!( + target: "sub-libp2p", + "Failed to push message to queue, dropped it" + ); + } + } else { + // This is an API misuse. + log::warn!( + target: "sub-libp2p", + "Tried to send a notification on a disabled handler" + ); + }, + } + } + + fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr) { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled => {}, + State::DisabledOpen(_) | State::Refused | State::Open { .. } => + error!("State mismatch in NotificationsOut"), + State::Opening { .. } => { + self.state = State::Refused; + let ev = NotifsOutHandlerOut::Refused; + self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + }, + State::DisabledOpening => self.state = State::Disabled, + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + fn connection_keep_alive(&self) -> KeepAlive { + match self.state { + // We have a small grace period of `INITIAL_KEEPALIVE_TIME` during which we keep the + // connection open no matter what, in order to avoid closing and reopening + // connections all the time. + State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => + KeepAlive::Until(self.when_connection_open + INITIAL_KEEPALIVE_TIME), + State::Opening { .. } | State::Open { .. } => KeepAlive::Yes, + State::Refused | State::Poisoned => KeepAlive::No, + } + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll> { + // Flush the events queue if necessary. + if !self.events_queue.is_empty() { + let event = self.events_queue.remove(0); + return Poll::Ready(event); + } + + match &mut self.state { + State::Open { substream, initial_message } => + match Sink::poll_flush(Pin::new(substream), cx) { + Poll::Pending | Poll::Ready(Ok(())) => {}, + Poll::Ready(Err(_)) => { + // We try to re-open a substream. + let initial_message = mem::replace(initial_message, Vec::new()); + self.state = State::Opening { initial_message: initial_message.clone() }; + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed)); + } + }, + + State::DisabledOpen(sub) => match Sink::poll_close(Pin::new(sub), cx) { + Poll::Pending => {}, + Poll::Ready(Ok(())) | Poll::Ready(Err(_)) => { + self.state = State::Disabled; + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed)); + }, + }, + + _ => {} + } + + Poll::Pending + } +} + +impl fmt::Debug for NotifsOutHandler { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.debug_struct("NotifsOutHandler") + .field("open", &self.is_open()) + .finish() + } +} diff --git a/client/network/src/protocol/legacy_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs similarity index 85% rename from client/network/src/protocol/legacy_proto/tests.rs rename to client/network/src/protocol/generic_proto/tests.rs index 18e32f1d0189fa89b6863e55c17167f42188427f..b8436e2c7f704bf2c4d5664c6908fce96c80d65c 100644 --- a/client/network/src/protocol/legacy_proto/tests.rs +++ b/client/network/src/protocol/generic_proto/tests.rs @@ -18,24 +18,21 @@ use futures::{prelude::*, ready}; use codec::{Encode, Decode}; -use libp2p::core::nodes::{Substream, listeners::ListenerId}; -use libp2p::core::{ConnectedPoint, transport::boxed::Boxed, muxing::StreamMuxerBox}; +use libp2p::core::nodes::listeners::ListenerId; +use libp2p::core::ConnectedPoint; use libp2p::swarm::{Swarm, ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction}; use libp2p::{PeerId, Multiaddr, Transport}; use rand::seq::SliceRandom; use std::{error, io, task::Context, task::Poll, time::Duration}; -use crate::message::Message; -use crate::protocol::legacy_proto::{LegacyProto, LegacyProtoOut}; +use std::collections::HashSet; +use crate::message::{generic::BlockResponse, Message}; +use crate::protocol::generic_proto::{GenericProto, GenericProtoOut}; use sp_test_primitives::Block; /// Builds two nodes that have each other as bootstrap nodes. /// This is to be used only for testing, and a panic will happen if something goes wrong. -fn build_nodes() --> ( - Swarm, CustomProtoWithAddr>, - Swarm, CustomProtoWithAddr> -) { +fn build_nodes() -> (Swarm, Swarm) { let mut out = Vec::with_capacity(2); let keypairs: Vec<_> = (0..2).map(|_| libp2p::identity::Keypair::generate_ed25519()).collect(); @@ -85,7 +82,7 @@ fn build_nodes() }); let behaviour = CustomProtoWithAddr { - inner: LegacyProto::new(&b"test"[..], &[1], peerset), + inner: GenericProto::new(&b"test"[..], &[1], peerset), addrs: addrs .iter() .enumerate() @@ -115,12 +112,12 @@ fn build_nodes() /// Wraps around the `CustomBehaviour` network behaviour, and adds hardcoded node addresses to it. struct CustomProtoWithAddr { - inner: LegacyProto>, + inner: GenericProto, addrs: Vec<(PeerId, Multiaddr)>, } impl std::ops::Deref for CustomProtoWithAddr { - type Target = LegacyProto>; + type Target = GenericProto; fn deref(&self) -> &Self::Target { &self.inner @@ -134,9 +131,8 @@ impl std::ops::DerefMut for CustomProtoWithAddr { } impl NetworkBehaviour for CustomProtoWithAddr { - type ProtocolsHandler = - > as NetworkBehaviour>::ProtocolsHandler; - type OutEvent = > as NetworkBehaviour>::OutEvent; + type ProtocolsHandler = ::ProtocolsHandler; + type OutEvent = ::OutEvent; fn new_handler(&mut self) -> Self::ProtocolsHandler { self.inner.new_handler() @@ -228,11 +224,14 @@ fn two_nodes_transfer_lots_of_packets() { let fut1 = future::poll_fn(move |cx| -> Poll<()> { loop { match ready!(service1.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => { for n in 0 .. NUM_PACKETS { service1.send_packet( &peer_id, - Message::::ChainSpecific(vec![(n % 256) as u8]).encode() + Message::::BlockResponse(BlockResponse { + id: n as _, + blocks: Vec::new(), + }).encode() ); } }, @@ -245,11 +244,11 @@ fn two_nodes_transfer_lots_of_packets() { let fut2 = future::poll_fn(move |cx| { loop { match ready!(service2.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {}, - Some(LegacyProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, + Some(GenericProtoOut::CustomMessage { message, .. }) => { match Message::::decode(&mut &message[..]).unwrap() { - Message::::ChainSpecific(message) => { - assert_eq!(message.len(), 1); + Message::::BlockResponse(BlockResponse { id: _, blocks }) => { + assert!(blocks.is_empty()); packet_counter += 1; if packet_counter == NUM_PACKETS { return Poll::Ready(()) @@ -275,9 +274,21 @@ fn basic_two_nodes_requests_in_parallel() { // Generate random messages with or without a request id. let mut to_send = { let mut to_send = Vec::new(); + let mut existing_ids = HashSet::new(); for _ in 0..200 { // Note: don't make that number too high or the CPU usage will explode. - let msg = (0..10).map(|_| rand::random::()).collect::>(); - to_send.push(Message::::ChainSpecific(msg)); + let req_id = loop { + let req_id = rand::random::(); + + // ensure uniqueness - odds of randomly sampling collisions + // is unlikely, but possible to cause spurious test failures. + if existing_ids.insert(req_id) { + break req_id; + } + }; + + to_send.push(Message::::BlockResponse( + BlockResponse { id: req_id, blocks: Vec::new() } + )); } to_send }; @@ -290,7 +301,7 @@ fn basic_two_nodes_requests_in_parallel() { let fut1 = future::poll_fn(move |cx| -> Poll<()> { loop { match ready!(service1.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => { for msg in to_send.drain(..) { service1.send_packet(&peer_id, msg.encode()); } @@ -303,8 +314,8 @@ fn basic_two_nodes_requests_in_parallel() { let fut2 = future::poll_fn(move |cx| { loop { match ready!(service2.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {}, - Some(LegacyProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, + Some(GenericProtoOut::CustomMessage { message, .. }) => { let pos = to_receive.iter().position(|m| m.encode() == message).unwrap(); to_receive.remove(pos); if to_receive.is_empty() { @@ -340,7 +351,7 @@ fn reconnect_after_disconnect() { let mut service1_not_ready = false; match service1.poll_next_unpin(cx) { - Poll::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolOpen { .. })) => { match service1_state { ServiceState::NotConnected => { service1_state = ServiceState::FirstConnec; @@ -352,7 +363,7 @@ fn reconnect_after_disconnect() { ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(), } }, - Poll::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolClosed { .. })) => { match service1_state { ServiceState::FirstConnec => service1_state = ServiceState::Disconnected, ServiceState::ConnectedAgain| ServiceState::NotConnected | @@ -364,7 +375,7 @@ fn reconnect_after_disconnect() { } match service2.poll_next_unpin(cx) { - Poll::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolOpen { .. })) => { match service2_state { ServiceState::NotConnected => { service2_state = ServiceState::FirstConnec; @@ -376,7 +387,7 @@ fn reconnect_after_disconnect() { ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(), } }, - Poll::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolClosed { .. })) => { match service2_state { ServiceState::FirstConnec => service2_state = ServiceState::Disconnected, ServiceState::ConnectedAgain| ServiceState::NotConnected | @@ -409,7 +420,7 @@ fn reconnect_after_disconnect() { _ => panic!() } - if let Poll::Ready(Ok(_)) = delay.poll_unpin(cx) { + if let Poll::Ready(()) = delay.poll_unpin(cx) { Poll::Ready(Ok(())) } else { Poll::Pending diff --git a/client/network/src/protocol/generic_proto/upgrade.rs b/client/network/src/protocol/generic_proto/upgrade.rs new file mode 100644 index 0000000000000000000000000000000000000000..36f826336532619479b94876776c65e8636f8c57 --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade.rs @@ -0,0 +1,35 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +pub use self::collec::UpgradeCollec; +pub use self::legacy::{ + RegisteredProtocol, + RegisteredProtocolEvent, + RegisteredProtocolName, + RegisteredProtocolSubstream +}; +pub use self::notifications::{ + NotificationsIn, + NotificationsInSubstream, + NotificationsOut, + NotificationsOutSubstream, + NotificationsHandshakeError, + NotificationsOutError, +}; + +mod collec; +mod legacy; +mod notifications; diff --git a/client/network/src/protocol/generic_proto/upgrade/collec.rs b/client/network/src/protocol/generic_proto/upgrade/collec.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8d199974940fb1abe6d41829d728243dd73af23 --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade/collec.rs @@ -0,0 +1,97 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use futures::prelude::*; +use libp2p::core::upgrade::{InboundUpgrade, ProtocolName, UpgradeInfo}; +use std::{iter::FromIterator, pin::Pin, task::{Context, Poll}, vec}; + +// TODO: move this to libp2p => https://github.com/libp2p/rust-libp2p/issues/1445 + +/// Upgrade that combines multiple upgrades of the same type into one. Supports all the protocols +/// supported by either sub-upgrade. +#[derive(Debug, Clone)] +pub struct UpgradeCollec(pub Vec); + +impl From> for UpgradeCollec { + fn from(list: Vec) -> Self { + UpgradeCollec(list) + } +} + +impl FromIterator for UpgradeCollec { + fn from_iter>(iter: I) -> Self { + UpgradeCollec(iter.into_iter().collect()) + } +} + +impl UpgradeInfo for UpgradeCollec { + type Info = ProtoNameWithUsize; + type InfoIter = vec::IntoIter; + + fn protocol_info(&self) -> Self::InfoIter { + self.0.iter().enumerate() + .flat_map(|(n, p)| + p.protocol_info().into_iter().map(move |i| ProtoNameWithUsize(i, n))) + .collect::>() + .into_iter() + } +} + +impl InboundUpgrade for UpgradeCollec +where + T: InboundUpgrade, +{ + type Output = (T::Output, usize); + type Error = (T::Error, usize); + type Future = FutWithUsize; + + fn upgrade_inbound(mut self, sock: C, info: Self::Info) -> Self::Future { + let fut = self.0.remove(info.1).upgrade_inbound(sock, info.0); + FutWithUsize(fut, info.1) + } +} + +/// Groups a `ProtocolName` with a `usize`. +#[derive(Debug, Clone)] +pub struct ProtoNameWithUsize(T, usize); + +impl ProtocolName for ProtoNameWithUsize { + fn protocol_name(&self) -> &[u8] { + self.0.protocol_name() + } +} + +/// Equivalent to `fut.map_ok(|v| (v, num)).map_err(|e| (e, num))`, where `fut` and `num` are +/// the two fields of this struct. +#[pin_project::pin_project] +pub struct FutWithUsize(#[pin] T, usize); + +impl>, O, E> Future for FutWithUsize { + type Output = Result<(O, usize), (E, usize)>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = self.project(); + match Future::poll(this.0, cx) { + Poll::Ready(Ok(v)) => Poll::Ready(Ok((v, *this.1))), + Poll::Ready(Err(e)) => Poll::Ready(Err((e, *this.1))), + Poll::Pending => Poll::Pending, + } + } +} diff --git a/client/network/src/protocol/legacy_proto/upgrade.rs b/client/network/src/protocol/generic_proto/upgrade/legacy.rs similarity index 100% rename from client/network/src/protocol/legacy_proto/upgrade.rs rename to client/network/src/protocol/generic_proto/upgrade/legacy.rs diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs new file mode 100644 index 0000000000000000000000000000000000000000..ddc07b5d6f3d6b2ffefe8fec47cee78768a1495a --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs @@ -0,0 +1,622 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// Notifications protocol. +/// +/// The Substrate notifications protocol consists in the following: +/// +/// - Node A opens a substream to node B and sends a message which contains some protocol-specific +/// higher-level logic. This message is prefixed with a variable-length integer message length. +/// This message can be empty, in which case `0` is sent. +/// - If node B accepts the substream, it sends back a message with the same properties. +/// Afterwards, the sending side of B is closed. +/// - If instead B refuses the connection (which typically happens because no empty slot is +/// available), then it immediately closes the substream without sending back anything. +/// - Node A can then send notifications to B, prefixed with a variable-length integer indicating +/// the length of the message. +/// - Node A closes its writing side if it doesn't want the notifications substream anymore. +/// +/// Notification substreams are unidirectional. If A opens a substream with B, then B is +/// encouraged but not required to open a substream to A as well. +/// + +use bytes::BytesMut; +use futures::{prelude::*, ready}; +use futures_codec::Framed; +use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade}; +use log::error; +use std::{borrow::Cow, collections::VecDeque, io, iter, mem, pin::Pin, task::{Context, Poll}}; +use unsigned_varint::codec::UviBytes; + +/// Maximum allowed size of the two handshake messages, in bytes. +const MAX_HANDSHAKE_SIZE: usize = 1024; +/// Maximum number of buffered messages before we consider the remote unresponsive and kill the +/// substream. +const MAX_PENDING_MESSAGES: usize = 256; + +/// Upgrade that accepts a substream, sends back a status message, then becomes a unidirectional +/// stream of messages. +#[derive(Debug, Clone)] +pub struct NotificationsIn { + /// Protocol name to use when negotiating the substream. + protocol_name: Cow<'static, [u8]>, +} + +/// Upgrade that opens a substream, waits for the remote to accept by sending back a status +/// message, then becomes a unidirectional sink of data. +#[derive(Debug, Clone)] +pub struct NotificationsOut { + /// Protocol name to use when negotiating the substream. + protocol_name: Cow<'static, [u8]>, + /// Message to send when we start the handshake. + initial_message: Vec, +} + +/// A substream for incoming notification messages. +/// +/// When creating, this struct starts in a state in which we must first send back a handshake +/// message to the remote. No message will come before this has been done. +#[pin_project::pin_project] +pub struct NotificationsInSubstream { + #[pin] + socket: Framed>>>, + handshake: NotificationsInSubstreamHandshake, +} + +/// State of the handshake sending back process. +enum NotificationsInSubstreamHandshake { + /// Waiting for the user to give us the handshake message. + NotSent, + /// User gave us the handshake message. Trying to push it in the socket. + PendingSend(Vec), + /// Handshake message was pushed in the socket. Still need to flush. + Close, + /// Handshake message successfully sent. + Sent, +} + +/// A substream for outgoing notification messages. +#[pin_project::pin_project] +pub struct NotificationsOutSubstream { + /// Substream where to send messages. + #[pin] + socket: Framed>>>, + /// Queue of messages waiting to be sent. + messages_queue: VecDeque>, + /// If true, we need to flush `socket`. + need_flush: bool, +} + +impl NotificationsIn { + /// Builds a new potential upgrade. + pub fn new(protocol_name: impl Into>) -> Self { + NotificationsIn { + protocol_name: protocol_name.into(), + } + } + + /// Returns the name of the protocol that we accept. + pub fn protocol_name(&self) -> &[u8] { + &self.protocol_name + } +} + +impl UpgradeInfo for NotificationsIn { + type Info = Cow<'static, [u8]>; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol_name.clone()) + } +} + +impl InboundUpgrade for NotificationsIn +where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Output = (Vec, NotificationsInSubstream); + type Future = Pin> + Send>>; + type Error = NotificationsHandshakeError; + + fn upgrade_inbound( + self, + mut socket: TSubstream, + _: Self::Info, + ) -> Self::Future { + Box::pin(async move { + let initial_message_len = unsigned_varint::aio::read_usize(&mut socket).await?; + if initial_message_len > MAX_HANDSHAKE_SIZE { + return Err(NotificationsHandshakeError::TooLarge { + requested: initial_message_len, + max: MAX_HANDSHAKE_SIZE, + }); + } + + let mut initial_message = vec![0u8; initial_message_len]; + if !initial_message.is_empty() { + socket.read(&mut initial_message).await?; + } + + let substream = NotificationsInSubstream { + socket: Framed::new(socket, UviBytes::default()), + handshake: NotificationsInSubstreamHandshake::NotSent, + }; + + Ok((initial_message, substream)) + }) + } +} + +impl NotificationsInSubstream +where TSubstream: AsyncRead + AsyncWrite, +{ + /// Sends the handshake in order to inform the remote that we accept the substream. + pub fn send_handshake(&mut self, message: impl Into>) { + match self.handshake { + NotificationsInSubstreamHandshake::NotSent => {} + _ => { + error!(target: "sub-libp2p", "Tried to send handshake twice"); + return; + } + } + + self.handshake = NotificationsInSubstreamHandshake::PendingSend(message.into()); + } +} + +impl Stream for NotificationsInSubstream +where TSubstream: AsyncRead + AsyncWrite + Unpin, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut this = self.project(); + + // This `Stream` implementation first tries to send back the handshake if necessary. + loop { + match mem::replace(this.handshake, NotificationsInSubstreamHandshake::Sent) { + NotificationsInSubstreamHandshake::Sent => + return Stream::poll_next(this.socket.as_mut(), cx), + NotificationsInSubstreamHandshake::NotSent => + return Poll::Pending, + NotificationsInSubstreamHandshake::PendingSend(msg) => + match Sink::poll_ready(this.socket.as_mut(), cx) { + Poll::Ready(_) => { + *this.handshake = NotificationsInSubstreamHandshake::Close; + match Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg)) { + Ok(()) => {}, + Err(err) => return Poll::Ready(Some(Err(err))), + } + }, + Poll::Pending => + *this.handshake = NotificationsInSubstreamHandshake::PendingSend(msg), + }, + NotificationsInSubstreamHandshake::Close => + match Sink::poll_close(this.socket.as_mut(), cx)? { + Poll::Ready(()) => + *this.handshake = NotificationsInSubstreamHandshake::Sent, + Poll::Pending => + *this.handshake = NotificationsInSubstreamHandshake::Close, + }, + } + } + } +} + +impl NotificationsOut { + /// Builds a new potential upgrade. + pub fn new(protocol_name: impl Into>, initial_message: impl Into>) -> Self { + let initial_message = initial_message.into(); + if initial_message.len() > MAX_HANDSHAKE_SIZE { + error!(target: "sub-libp2p", "Outbound networking handshake is above allowed protocol limit"); + } + + NotificationsOut { + protocol_name: protocol_name.into(), + initial_message, + } + } +} + +impl UpgradeInfo for NotificationsOut { + type Info = Cow<'static, [u8]>; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol_name.clone()) + } +} + +impl OutboundUpgrade for NotificationsOut +where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Output = (Vec, NotificationsOutSubstream); + type Future = Pin> + Send>>; + type Error = NotificationsHandshakeError; + + fn upgrade_outbound( + self, + mut socket: TSubstream, + _: Self::Info, + ) -> Self::Future { + Box::pin(async move { + upgrade::write_with_len_prefix(&mut socket, &self.initial_message).await?; + + // Reading handshake. + let handshake_len = unsigned_varint::aio::read_usize(&mut socket).await?; + if handshake_len > MAX_HANDSHAKE_SIZE { + return Err(NotificationsHandshakeError::TooLarge { + requested: handshake_len, + max: MAX_HANDSHAKE_SIZE, + }); + } + + let mut handshake = vec![0u8; handshake_len]; + if !handshake.is_empty() { + socket.read(&mut handshake).await?; + } + + Ok((handshake, NotificationsOutSubstream { + socket: Framed::new(socket, UviBytes::default()), + messages_queue: VecDeque::with_capacity(MAX_PENDING_MESSAGES), + need_flush: false, + })) + }) + } +} + +impl Sink> for NotificationsOutSubstream + where TSubstream: AsyncRead + AsyncWrite + Unpin, +{ + type Error = NotificationsOutError; + + fn poll_ready(self: Pin<&mut Self>, _: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(mut self: Pin<&mut Self>, item: Vec) -> Result<(), Self::Error> { + if self.messages_queue.len() >= MAX_PENDING_MESSAGES { + return Err(NotificationsOutError::Clogged); + } + + self.messages_queue.push_back(item); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut this = self.project(); + + while !this.messages_queue.is_empty() { + match Sink::poll_ready(this.socket.as_mut(), cx) { + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + Poll::Ready(Ok(())) => { + let msg = this.messages_queue.pop_front() + .expect("checked for !is_empty above; qed"); + Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg))?; + *this.need_flush = true; + }, + Poll::Pending => return Poll::Pending, + } + } + + if *this.need_flush { + match Sink::poll_flush(this.socket.as_mut(), cx) { + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + Poll::Ready(Ok(())) => *this.need_flush = false, + Poll::Pending => return Poll::Pending, + } + } + + Poll::Ready(Ok(())) + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + ready!(Sink::poll_flush(self.as_mut(), cx))?; + let this = self.project(); + match Sink::poll_close(this.socket, cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(())), + Poll::Ready(Err(err)) => Poll::Ready(Err(From::from(err))), + Poll::Pending => Poll::Pending, + } + } +} + +/// Error generated by sending on a notifications out substream. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum NotificationsHandshakeError { + /// I/O error on the substream. + Io(io::Error), + + /// Initial message or handshake was too large. + #[display(fmt = "Initial message or handshake was too large: {}", requested)] + TooLarge { + /// Size requested by the remote. + requested: usize, + /// Maximum allowed, + max: usize, + }, + + /// Error while decoding the variable-length integer. + VarintDecode(unsigned_varint::decode::Error), +} + +impl From for NotificationsHandshakeError { + fn from(err: unsigned_varint::io::ReadError) -> Self { + match err { + unsigned_varint::io::ReadError::Io(err) => NotificationsHandshakeError::Io(err), + unsigned_varint::io::ReadError::Decode(err) => NotificationsHandshakeError::VarintDecode(err), + _ => { + log::warn!("Unrecognized varint decoding error"); + NotificationsHandshakeError::Io(From::from(io::ErrorKind::InvalidData)) + } + } + } +} + +/// Error generated by sending on a notifications out substream. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum NotificationsOutError { + /// I/O error on the substream. + Io(io::Error), + + /// Remote doesn't process our messages quickly enough. + /// + /// > **Note**: This is not necessarily the remote's fault, and could also be caused by the + /// > local node sending data too quickly. Properly doing back-pressure, however, + /// > would require a deep refactoring effort in Substrate as a whole. + Clogged, +} + +#[cfg(test)] +mod tests { + use super::{NotificationsIn, NotificationsOut}; + + use async_std::net::{TcpListener, TcpStream}; + use futures::{prelude::*, channel::oneshot}; + use libp2p::core::upgrade; + use std::pin::Pin; + + #[test] + fn basic_works() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"initial message"[..]), + upgrade::Version::V1 + ).await.unwrap(); + + assert_eq!(handshake, b"hello world"); + substream.send(b"test message".to_vec()).await.unwrap(); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert_eq!(initial_message, b"initial message"); + substream.send_handshake(&b"hello world"[..]); + + let msg = substream.next().await.unwrap().unwrap(); + assert_eq!(msg.as_ref(), b"test message"); + }); + + async_std::task::block_on(client); + } + + #[test] + fn empty_handshake() { + // Check that everything still works when the handshake messages are empty. + + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, vec![]), + upgrade::Version::V1 + ).await.unwrap(); + + assert!(handshake.is_empty()); + substream.send(Default::default()).await.unwrap(); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert!(initial_message.is_empty()); + substream.send_handshake(vec![]); + + let msg = substream.next().await.unwrap().unwrap(); + assert!(msg.as_ref().is_empty()); + }); + + async_std::task::block_on(client); + } + + #[test] + fn refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let outcome = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"hello"[..]), + upgrade::Version::V1 + ).await; + + // Despite the protocol negotiation being successfully conducted on the listener + // side, we have to receive an error here because the listener didn't send the + // handshake. + assert!(outcome.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_msg, substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert_eq!(initial_msg, b"hello"); + + // We successfully upgrade to the protocol, but then close the substream. + drop(substream); + }); + + async_std::task::block_on(client); + } + + #[test] + fn large_initial_message_refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let ret = upgrade::apply_outbound( + socket, + // We check that an initial message that is too large gets refused. + NotificationsOut::new(PROTO_NAME, (0..32768).map(|_| 0).collect::>()), + upgrade::Version::V1 + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let ret = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(client); + } + + #[test] + fn large_handshake_refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let ret = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"initial message"[..]), + upgrade::Version::V1 + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + assert_eq!(initial_message, b"initial message"); + + // We check that a handshake that is too large gets refused. + substream.send_handshake((0..32768).map(|_| 0).collect::>()); + let _ = substream.next().await; + }); + + async_std::task::block_on(client); + } + + #[test] + fn buffer_is_full_closes_connection() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, vec![]), + upgrade::Version::V1 + ).await.unwrap(); + + assert!(handshake.is_empty()); + + // Push an item and flush so that the test works. + substream.send(b"hello world".to_vec()).await.unwrap(); + + for _ in 0..32768 { + // Push an item on the sink without flushing until an error happens because the + // buffer is full. + let message = b"hello world!".to_vec(); + if future::poll_fn(|cx| Sink::poll_ready(Pin::new(&mut substream), cx)).await.is_err() { + return Ok(()); + } + if Sink::start_send(Pin::new(&mut substream), message).is_err() { + return Ok(()); + } + } + + Err(()) + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert!(initial_message.is_empty()); + substream.send_handshake(vec![]); + + // Process one message so that the handshake and all works. + let _ = substream.next().await.unwrap().unwrap(); + + client.await.unwrap(); + }); + } +} diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs new file mode 100644 index 0000000000000000000000000000000000000000..77cf71408d6c49f78511cb6c6936820eaa126c1e --- /dev/null +++ b/client/network/src/protocol/light_client_handler.rs @@ -0,0 +1,1793 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! [`NetworkBehaviour`] implementation which handles light client requests. +//! +//! Every request is coming in on a separate connection substream which gets +//! closed after we have sent the response back. Requests and responses are +//! encoded as protocol buffers (cf. `api.v1.proto`). +//! +//! For every outgoing request we likewise open a separate substream. + +#![allow(unused)] + +use bytes::Bytes; +use codec::{self, Encode, Decode}; +use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, light_dispatch::TIMEOUT_REPUTATION_CHANGE} +}; +use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use libp2p::{ + core::{ + ConnectedPoint, + Multiaddr, + PeerId, + upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + upgrade::{OutboundUpgrade, read_one, write_one} + }, + swarm::{ + NegotiatedSubstream, + NetworkBehaviour, + NetworkBehaviourAction, + OneShotHandler, + PollParameters, + SubstreamProtocol + } +}; +use nohash_hasher::IntMap; +use prost::Message; +use rustc_hex::ToHex; +use sc_client::light::fetcher; +use sc_client_api::StorageProof; +use sc_peerset::ReputationChange; +use sp_core::storage::{ChildInfo, StorageKey}; +use sp_blockchain::{Error as ClientError}; +use sp_runtime::traits::{Block, Header, NumberFor, Zero}; +use std::{ + collections::{BTreeMap, VecDeque, HashMap}, + iter, + io, + sync::Arc, + time::Duration, + task::{Context, Poll} +}; +use void::Void; +use wasm_timer::Instant; + +/// Configuration options for `LightClientHandler` behaviour. +#[derive(Debug, Clone)] +pub struct Config { + max_data_size: usize, + max_pending_requests: usize, + inactivity_timeout: Duration, + request_timeout: Duration, + protocol: Bytes, +} + +impl Config { + /// Create a fresh configuration with the following options: + /// + /// - max. data size = 1 MiB + /// - max. pending requests = 128 + /// - inactivity timeout = 15s + /// - request timeout = 15s + pub fn new(id: &ProtocolId) -> Self { + let mut c = Config { + max_data_size: 1024 * 1024, + max_pending_requests: 128, + inactivity_timeout: Duration::from_secs(15), + request_timeout: Duration::from_secs(15), + protocol: Bytes::new(), + }; + c.set_protocol(id); + c + } + + /// Limit the max. length of incoming request bytes. + pub fn set_max_data_size(&mut self, v: usize) -> &mut Self { + self.max_data_size = v; + self + } + + /// Limit the max. number of pending requests. + pub fn set_max_pending_requests(&mut self, v: usize) -> &mut Self { + self.max_pending_requests = v; + self + } + + /// Limit the max. duration the connection may remain inactive before closing it. + pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { + self.inactivity_timeout = v; + self + } + + /// Limit the max. request duration. + pub fn set_request_timeout(&mut self, v: Duration) -> &mut Self { + self.request_timeout = v; + self + } + + /// Set protocol to use for upgrade negotiation. + pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { + let mut v = Vec::new(); + v.extend_from_slice(b"/"); + v.extend_from_slice(id.as_bytes()); + v.extend_from_slice(b"/light/1"); + self.protocol = v.into(); + self + } +} + +/// Possible errors while handling light clients. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// There are currently too many pending request. + #[error("too many pending requests")] + TooManyRequests, + /// The response type does not correspond to the issued request. + #[error("unexpected response")] + UnexpectedResponse, + /// A bad request has been received. + #[error("bad request: {0}")] + BadRequest(&'static str), + /// The chain client errored. + #[error("client error: {0}")] + Client(#[from] ClientError), + /// Encoding or decoding of some data failed. + #[error("codec error: {0}")] + Codec(#[from] codec::Error), +} + +/// The possible light client requests we support. +/// +/// The associated `oneshot::Sender` will be used to convey the result of +/// their request back to them (cf. `Reply`). +// +// This is modeled after light_dispatch.rs's `RequestData` which is not +// used because we currently only support a subset of those. +#[derive(Debug)] +pub enum Request { + Header { + request: fetcher::RemoteHeaderRequest, + sender: oneshot::Sender> + }, + Read { + request: fetcher::RemoteReadRequest, + sender: oneshot::Sender, Option>>, ClientError>> + }, + ReadChild { + request: fetcher::RemoteReadChildRequest, + sender: oneshot::Sender, Option>>, ClientError>> + }, + Call { + request: fetcher::RemoteCallRequest, + sender: oneshot::Sender, ClientError>> + }, + Changes { + request: fetcher::RemoteChangesRequest, + sender: oneshot::Sender, u32)>, ClientError>> + } +} + +/// The data to send back to the light client over the oneshot channel. +// +// It is unified here in order to be able to return it as a function +// result instead of delivering it to the client as a side effect of +// response processing. +#[derive(Debug)] +enum Reply { + VecU8(Vec), + VecNumberU32(Vec<(::Number, u32)>), + MapVecU8OptVecU8(HashMap, Option>>), + Header(B::Header) +} + +/// Augments a light client request with metadata. +#[derive(Debug)] +struct RequestWrapper { + /// Time when this value was created. + timestamp: Instant, + /// Remaining retries. + retries: usize, + /// The actual request. + request: Request, + /// Peer information, e.g. `PeerId`. + peer: P +} + +/// Information we have about some peer. +#[derive(Debug)] +struct PeerInfo { + address: Multiaddr, + best_block: Option>, + status: PeerStatus, +} + +/// A peer is either idle or busy processing a request from us. +#[derive(Debug, Clone, PartialEq, Eq)] +enum PeerStatus { + /// The peer is available. + Idle, + /// We wait for the peer to return us a response for the given request ID. + BusyWith(u64), +} + +/// The light client handler behaviour. +pub struct LightClientHandler { + /// This behaviour's configuration. + config: Config, + /// Blockchain client. + chain: Arc>, + /// Verifies that received responses are correct. + checker: Arc>, + /// Peer information (addresses, their best block, etc.) + peers: HashMap>, + /// Futures sending back response to remote clients. + responses: FuturesUnordered>, + /// Pending (local) requests. + pending_requests: VecDeque>, + /// Requests on their way to remote peers. + outstanding: IntMap>, + /// (Local) Request ID counter + next_request_id: u64, + /// Handle to use for reporting misbehaviour of peers. + peerset: sc_peerset::PeersetHandle, +} + +impl LightClientHandler +where + B: Block, +{ + /// Construct a new light client handler. + pub fn new + ( cfg: Config + , chain: Arc> + , checker: Arc> + , peerset: sc_peerset::PeersetHandle + ) -> Self + { + LightClientHandler { + config: cfg, + chain, + checker, + peers: HashMap::new(), + responses: FuturesUnordered::new(), + pending_requests: VecDeque::new(), + outstanding: IntMap::default(), + next_request_id: 1, + peerset, + } + } + + /// We rely on external information about peers best blocks as we lack the + /// means to determine it ourselves. + pub fn update_best_block(&mut self, peer: &PeerId, num: NumberFor) { + if let Some(info) = self.peers.get_mut(peer) { + info.best_block = Some(num) + } + } + + /// Issue a new light client request. + pub fn request(&mut self, req: Request) -> Result<(), Error> { + if self.pending_requests.len() >= self.config.max_pending_requests { + return Err(Error::TooManyRequests) + } + let rw = RequestWrapper { + timestamp: Instant::now(), + retries: retries(&req), + request: req, + peer: (), // we do not know the peer yet + }; + self.pending_requests.push_back(rw); + Ok(()) + } + + fn next_request_id(&mut self) -> u64 { + let id = self.next_request_id; + self.next_request_id += 1; + id + } + + // Iterate over peers known to possess a certain block. + fn idle_peers_with_block(&mut self, num: NumberFor) -> impl Iterator + '_ { + self.peers.iter() + .filter(move |(_, info)| { + info.status == PeerStatus::Idle && info.best_block >= Some(num) + }) + .map(|(peer, _)| peer.clone()) + } + + // Iterate over peers without a known block. + fn idle_peers_with_unknown_block(&mut self) -> impl Iterator + '_ { + self.peers.iter() + .filter(|(_, info)| { + info.status == PeerStatus::Idle && info.best_block.is_none() + }) + .map(|(peer, _)| peer.clone()) + } + + /// Remove the given peer. + /// + /// If we have a request to this peer in flight, we move it back to + /// the pending requests queue. + fn remove_peer(&mut self, peer: &PeerId) { + if let Some(id) = self.outstanding.iter().find(|(_, rw)| &rw.peer == peer).map(|(k, _)| *k) { + let rw = self.outstanding.remove(&id).expect("key belongs to entry in this map"); + let rw = RequestWrapper { + timestamp: rw.timestamp, + retries: rw.retries, + request: rw.request, + peer: (), // need to find another peer + }; + self.pending_requests.push_back(rw); + } + self.peers.remove(peer); + } + + /// Process a local request's response from remote. + /// + /// If successful, this will give us the actual, checked data we should be + /// sending back to the client, otherwise an error. + fn on_response + ( &mut self + , peer: &PeerId + , request: &Request + , response: api::v1::light::Response + ) -> Result, Error> + { + log::trace!("response {} from {}", response.id, peer); + use api::v1::light::response::Response; + match response.response { + Some(Response::RemoteCallResponse(response)) => + if let Request::Call { request , .. } = request { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_execution_proof(request, proof)?; + Ok(Reply::VecU8(reply)) + } else { + Err(Error::UnexpectedResponse) + } + Some(Response::RemoteReadResponse(response)) => + match request { + Request::Read { request, .. } => { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_read_proof(&request, proof)?; + Ok(Reply::MapVecU8OptVecU8(reply)) + } + Request::ReadChild { request, .. } => { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_read_child_proof(&request, proof)?; + Ok(Reply::MapVecU8OptVecU8(reply)) + } + _ => Err(Error::UnexpectedResponse) + } + Some(Response::RemoteChangesResponse(response)) => + if let Request::Changes { request, .. } = request { + let max_block = Decode::decode(&mut response.max.as_ref())?; + let roots_proof = Decode::decode(&mut response.roots_proof.as_ref())?; + let roots = { + let mut r = BTreeMap::new(); + for pair in response.roots { + let k = Decode::decode(&mut pair.fst.as_ref())?; + let v = Decode::decode(&mut pair.snd.as_ref())?; + r.insert(k, v); + } + r + }; + let reply = self.checker.check_changes_proof(&request, fetcher::ChangesProof { + max_block, + proof: response.proof, + roots, + roots_proof, + })?; + Ok(Reply::VecNumberU32(reply)) + } else { + Err(Error::UnexpectedResponse) + } + Some(Response::RemoteHeaderResponse(response)) => + if let Request::Header { request, .. } = request { + let header = + if response.header.is_empty() { + None + } else { + Some(Decode::decode(&mut response.header.as_ref())?) + }; + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_header_proof(&request, header, proof)?; + Ok(Reply::Header(reply)) + } else { + Err(Error::UnexpectedResponse) + } + None => Err(Error::UnexpectedResponse) + } + } + + fn on_remote_call_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteCallRequest + ) -> Result + { + log::trace!("remote call request {} from {} ({} at {:?})", + request_id, + peer, + request.method, + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = match self.chain.execution_proof(&block, &request.method, &request.data) { + Ok((_, proof)) => proof, + Err(e) => { + log::trace!("remote call request {} from {} ({} at {:?}) failed with: {}", + request_id, + peer, + request.method, + request.block, + e); + StorageProof::empty() + } + }; + + let response = { + let r = api::v1::light::RemoteCallResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteCallResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_read_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteReadRequest + ) -> Result + { + if request.keys.is_empty() { + log::debug!("invalid remote read request sent by {}", peer); + return Err(Error::BadRequest("remote read request without keys")) + } + + log::trace!("remote read request {} from {} ({} at {:?})", + request_id, + peer, + fmt_keys(request.keys.first(), request.keys.last()), + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = match self.chain.read_proof(&block, &request.keys) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote read request {} from {} ({} at {:?}) failed with: {}", + request_id, + peer, + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + error); + StorageProof::empty() + } + }; + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteReadResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_read_child_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteReadChildRequest + ) -> Result + { + if request.keys.is_empty() { + log::debug!("invalid remote child read request sent by {}", peer); + return Err(Error::BadRequest("remove read child request without keys")) + } + + log::trace!("remote read child request {} from {} ({} {} at {:?})", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = + if let Some(info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { + match self.chain.read_child_proof(&block, &request.storage_key, info, &request.keys) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + error); + StorageProof::empty() + } + } + } else { + log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + "invalid child info and type" + ); + StorageProof::empty() + }; + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteReadResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_header_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteHeaderRequest + ) -> Result + { + log::trace!("remote header proof request {} from {} ({:?})", request_id, peer, request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let (header, proof) = match self.chain.header_proof(block) { + Ok((header, proof)) => (header.encode(), proof), + Err(error) => { + log::trace!("remote header proof request {} from {} ({:?}) failed with: {}", + request_id, + peer, + request.block, + error); + (Default::default(), StorageProof::empty()) + } + }; + + let response = { + let r = api::v1::light::RemoteHeaderResponse { header, proof: proof.encode() }; + api::v1::light::response::Response::RemoteHeaderResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_changes_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteChangesRequest + ) -> Result + { + log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?})", + request_id, + peer, + if !request.storage_key.is_empty() { + format!("{} : {}", request.storage_key.to_hex::(), request.key.to_hex::()) + } else { + request.key.to_hex::() + }, + request.first, + request.last); + + let first = Decode::decode(&mut request.first.as_ref())?; + let last = Decode::decode(&mut request.last.as_ref())?; + let min = Decode::decode(&mut request.min.as_ref())?; + let max = Decode::decode(&mut request.max.as_ref())?; + let key = StorageKey(request.key.clone()); + let storage_key = + if request.storage_key.is_empty() { + None + } else { + Some(StorageKey(request.storage_key.clone())) + }; + + let proof = match self.chain.key_changes_proof(first, last, min, max, storage_key.as_ref(), &key) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?}) failed with: {}", + request_id, + peer, + if let Some(sk) = storage_key { + format!("{} : {}", sk.0.to_hex::(), key.0.to_hex::()) + } else { + key.0.to_hex::() + }, + request.first, + request.last, + error); + + fetcher::ChangesProof:: { + max_block: Zero::zero(), + proof: Vec::new(), + roots: BTreeMap::new(), + roots_proof: StorageProof::empty(), + } + } + }; + + let response = { + let r = api::v1::light::RemoteChangesResponse { + max: proof.max_block.encode(), + proof: proof.proof, + roots: proof.roots.into_iter() + .map(|(k, v)| api::v1::light::Pair { fst: k.encode(), snd: v.encode() }) + .collect(), + roots_proof: proof.roots_proof.encode(), + }; + api::v1::light::response::Response::RemoteChangesResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } +} + +impl NetworkBehaviour for LightClientHandler +where + B: Block +{ + type ProtocolsHandler = OneShotHandler>; + type OutEvent = Void; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + let p = InboundProtocol { + max_data_size: self.config.max_data_size, + protocol: self.config.protocol.clone(), + }; + OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + } + + fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec { + self.peers.get(peer) + .map(|info| vec![info.address.clone()]) + .unwrap_or_default() + } + + fn inject_connected(&mut self, peer: PeerId, info: ConnectedPoint) { + let peer_address = match info { + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address } => address + }; + + log::trace!("peer {} connected with address {}", peer, peer_address); + + let info = PeerInfo { + address: peer_address, + best_block: None, + status: PeerStatus::Idle, + }; + + self.peers.insert(peer, info); + } + + fn inject_disconnected(&mut self, peer: &PeerId, _: ConnectedPoint) { + log::trace!("peer {} disconnected", peer); + self.remove_peer(peer) + } + + fn inject_node_event(&mut self, peer: PeerId, event: Event) { + match event { + // An incoming request from remote has been received. + Event::Request(request, mut stream) => { + log::trace!("incoming request {} from {}", peer, request.id); + let result = match &request.request { + Some(api::v1::light::request::Request::RemoteCallRequest(r)) => + self.on_remote_call_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteReadRequest(r)) => + self.on_remote_read_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteHeaderRequest(r)) => + self.on_remote_header_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteReadChildRequest(r)) => + self.on_remote_read_child_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteChangesRequest(r)) => + self.on_remote_changes_request(&peer, request.id, r), + None => { + log::debug!("ignoring request {} without request data from peer {}", request.id, peer); + return + } + }; + match result { + Ok(response) => { + log::trace!("enqueueing response {} for peer {}", response.id, peer); + let mut data = Vec::new(); + if let Err(e) = response.encode(&mut data) { + log::debug!("error encoding response {} for peer {}: {}", response.id, peer, e) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!("error writing response: {}", e) + } + }; + self.responses.push(future.boxed()) + } + } + Err(Error::BadRequest(_)) => { + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new(-(1 << 12), "bad request")) + } + Err(e) => log::debug!("error handling request {} from peer {}: {}", request.id, peer, e) + } + } + // A response to one of our own requests has been received. + Event::Response(response) => { + let id = response.id; + if let Some(request) = self.outstanding.remove(&id) { + // We first just check if the response originates from the expected peer. + if request.peer != peer { + log::debug!("was expecting response {} from {} instead of {}", id, request.peer, peer); + self.outstanding.insert(id, request); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); + return + } + + if let Some(info) = self.peers.get_mut(&peer) { + if info.status != PeerStatus::BusyWith(id) { + // If we get here, something is wrong with our internal handling of peer + // status information. At any time, a single peer processes at most one + // request from us and its status should contain the request ID we are + // expecting a response for. If a peer would send us a response with a + // random ID, we should not have an entry for it with this peer ID in + // our `outstanding` map, so a malicious peer should not be able to get + // us here. It is our own fault and must be fixed! + panic!("unexpected peer status {:?} for {}", info.status, peer); + } + + info.status = PeerStatus::Idle; // Make peer available again. + + match self.on_response(&peer, &request.request, response) { + Ok(reply) => send_reply(Ok(reply), request.request), + Err(Error::UnexpectedResponse) => { + log::debug!("unexpected response {} from peer {}", id, peer); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("unexpected response from peer")); + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries, + request: request.request, + peer: (), + }; + self.pending_requests.push_back(rw); + } + Err(other) => { + log::debug!("error handling response {} from peer {}: {}", id, peer, other); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("invalid response from peer")); + if request.retries > 0 { + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries - 1, + request: request.request, + peer: (), + }; + self.pending_requests.push_back(rw) + } else { + send_reply(Err(ClientError::RemoteFetchFailed), request.request) + } + } + } + } else { + // If we get here, something is wrong with our internal handling of peers. + // We apparently have an entry in our `outstanding` map and the peer is the one we + // expected. So, if we can not find an entry for it in our peer information table, + // then these two collections are out of sync which must not happen and is a clear + // programmer error that must be fixed! + panic!("missing peer information for {}; response {}", peer, id); + } + } else { + log::debug!("unexpected response {} from peer {}", id, peer); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); + } + } + } + } + + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) -> Poll> { + // Process response sending futures. + while let Poll::Ready(Some(_)) = self.responses.poll_next_unpin(cx) {} + + // If we have a pending request to send, try to find an available peer and send it. + let now = Instant::now(); + while let Some(mut request) = self.pending_requests.pop_front() { + if now > request.timestamp + self.config.request_timeout { + if request.retries == 0 { + send_reply(Err(ClientError::RemoteFetchFailed), request.request); + continue + } + request.timestamp = Instant::now(); + request.retries -= 1 + } + let number = required_block(&request.request); + let available_peer = { + let p = self.idle_peers_with_block(number).next(); + if p.is_none() { + self.idle_peers_with_unknown_block().next() + } else { + p + } + }; + if let Some(peer) = available_peer { + let id = self.next_request_id(); + let rq = serialize_request(id, &request.request); + let mut buf = Vec::with_capacity(rq.encoded_len()); + if let Err(e) = rq.encode(&mut buf) { + log::debug!("failed to serialize request {}: {}", id, e); + send_reply(Err(ClientError::RemoteFetchFailed), request.request) + } else { + log::trace!("sending request {} to peer {}", id, peer); + let protocol = OutboundProtocol { + request: buf, + max_data_size: self.config.max_data_size, + protocol: self.config.protocol.clone(), + }; + self.peers.get_mut(&peer).map(|info| info.status = PeerStatus::BusyWith(id)); + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries, + request: request.request, + peer: peer.clone(), + }; + self.outstanding.insert(id, rw); + return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id: peer, event: protocol }) + } + } else { + self.pending_requests.push_front(request); + log::debug!("no peer available to send request to"); + break + } + } + + // Look for ongoing requests that have timed out. + let mut expired = Vec::new(); + for (id, rw) in &self.outstanding { + if now > rw.timestamp + self.config.request_timeout { + log::debug!("request {} timed out", id); + expired.push(*id) + } + } + for id in expired { + if let Some(rw) = self.outstanding.remove(&id) { + self.remove_peer(&rw.peer); + self.peerset.report_peer(rw.peer.clone(), + ReputationChange::new(TIMEOUT_REPUTATION_CHANGE, "light request timeout")); + if rw.retries == 0 { + send_reply(Err(ClientError::RemoteFetchFailed), rw.request); + continue + } + let rw = RequestWrapper { + timestamp: Instant::now(), + retries: rw.retries - 1, + request: rw.request, + peer: (), + }; + self.pending_requests.push_back(rw) + } + } + + Poll::Pending + } +} + +fn required_block(request: &Request) -> NumberFor { + match request { + Request::Header { request, .. } => request.block, + Request::Read { request, .. } => *request.header.number(), + Request::ReadChild { request, .. } => *request.header.number(), + Request::Call { request, .. } => *request.header.number(), + Request::Changes { request, .. } => request.max_block.0, + } +} + +fn retries(request: &Request) -> usize { + let rc = match request { + Request::Header { request, .. } => request.retry_count, + Request::Read { request, .. } => request.retry_count, + Request::ReadChild { request, .. } => request.retry_count, + Request::Call { request, .. } => request.retry_count, + Request::Changes { request, .. } => request.retry_count, + }; + rc.unwrap_or(0) +} + +fn serialize_request(id: u64, request: &Request) -> api::v1::light::Request { + let request = match request { + Request::Header { request, .. } => { + let r = api::v1::light::RemoteHeaderRequest { block: request.block.encode() }; + api::v1::light::request::Request::RemoteHeaderRequest(r) + } + Request::Read { request, .. } => { + let r = api::v1::light::RemoteReadRequest { + block: request.block.encode(), + keys: request.keys.clone(), + }; + api::v1::light::request::Request::RemoteReadRequest(r) + } + Request::ReadChild { request, .. } => { + let r = api::v1::light::RemoteReadChildRequest { + block: request.block.encode(), + storage_key: request.storage_key.clone(), + child_type: request.child_type.clone(), + child_info: request.child_info.clone(), + keys: request.keys.clone(), + }; + api::v1::light::request::Request::RemoteReadChildRequest(r) + } + Request::Call { request, .. } => { + let r = api::v1::light::RemoteCallRequest { + block: request.block.encode(), + method: request.method.clone(), + data: request.call_data.clone(), + }; + api::v1::light::request::Request::RemoteCallRequest(r) + } + Request::Changes { request, .. } => { + let r = api::v1::light::RemoteChangesRequest { + first: request.first_block.1.encode(), + last: request.last_block.1.encode(), + min: request.tries_roots.1.encode(), + max: request.max_block.1.encode(), + storage_key: request.storage_key.clone().unwrap_or_default(), + key: request.key.clone(), + }; + api::v1::light::request::Request::RemoteChangesRequest(r) + } + }; + + api::v1::light::Request { id, request: Some(request) } +} + +fn send_reply(result: Result, ClientError>, request: Request) { + fn send(item: T, sender: oneshot::Sender) { + let _ = sender.send(item); // It is okay if the other end already hung up. + } + match request { + Request::Header { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::Header(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for header request: {:?}, {:?}", reply, request), + } + Request::Read { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for read request: {:?}, {:?}", reply, request), + } + Request::ReadChild { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for read child request: {:?}, {:?}", reply, request), + } + Request::Call { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::VecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for call request: {:?}, {:?}", reply, request), + } + Request::Changes { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::VecNumberU32(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for changes request: {:?}, {:?}", reply, request), + } + } +} + +/// Output type of inbound and outbound substream upgrades. +#[derive(Debug)] +pub enum Event { + /// Incoming request from remote and substream to use for the response. + Request(api::v1::light::Request, T), + /// Incoming response from remote. + Response(api::v1::light::Response), +} + +/// Substream upgrade protocol. +/// +/// Reads incoming requests from remote. +#[derive(Debug, Clone)] +pub struct InboundProtocol { + /// The max. request length in bytes. + max_data_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for InboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl InboundUpgrade for InboundProtocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + let vec = read_one(&mut s, self.max_data_size).await?; + match api::v1::light::Request::decode(&vec[..]) { + Ok(r) => Ok(Event::Request(r, s)), + Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) + } + }; + future.boxed() + } +} + +/// Substream upgrade protocol. +/// +/// Sends a request to remote and awaits the response. +#[derive(Debug, Clone)] +pub struct OutboundProtocol { + /// The serialized protobuf request. + request: Vec, + /// The max. request length in bytes. + max_data_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for OutboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl OutboundUpgrade for OutboundProtocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + write_one(&mut s, &self.request).await?; + let vec = read_one(&mut s, self.max_data_size).await?; + api::v1::light::Response::decode(&vec[..]) + .map(Event::Response) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + }; + future.boxed() + } +} + +fn fmt_keys(first: Option<&Vec>, last: Option<&Vec>) -> String { + if let (Some(first), Some(last)) = (first, last) { + if first == last { + first.to_hex::() + } else { + format!("{}..{}", first.to_hex::(), last.to_hex::()) + } + } else { + String::from("n/a") + } +} + +#[cfg(test)] +mod tests { + use async_std::task; + use assert_matches::assert_matches; + use codec::Encode; + use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, light_dispatch::tests::{DummyFetchChecker, dummy_header}} + }; + use futures::{channel::oneshot, prelude::*}; + use libp2p::{ + PeerId, + Multiaddr, + core::{ + ConnectedPoint, + identity, + muxing::{StreamMuxerBox, SubstreamRef}, + transport::{Transport, boxed::Boxed, memory::MemoryTransport}, + upgrade + }, + noise::{self, Keypair, X25519, NoiseConfig}, + swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}, + yamux + }; + use sc_client_api::StorageProof; + use sc_client::light::fetcher; + use sp_blockchain::{Error as ClientError}; + use sp_core::storage::ChildInfo; + use std::{ + collections::HashSet, + io, + iter::{self, FromIterator}, + pin::Pin, + sync::Arc, + task::{Context, Poll} + }; + use sp_runtime::{generic::Header, traits::BlakeTwo256}; + use super::{Event, LightClientHandler, Request, OutboundProtocol, PeerStatus}; + use void::Void; + + const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"foobarbaz"); + + type Block = sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; + type Handler = LightClientHandler; + type Swarm = libp2p::swarm::Swarm; + + fn empty_proof() -> Vec { + StorageProof::empty().encode() + } + + fn make_swarm(ok: bool, ps: sc_peerset::PeersetHandle, cf: super::Config) -> Swarm { + let client = Arc::new(substrate_test_runtime_client::new()); + let checker = Arc::new(DummyFetchChecker::new(ok)); + let id_key = identity::Keypair::generate_ed25519(); + let dh_key = Keypair::::new().into_authentic(&id_key).unwrap(); + let local_peer = id_key.public().into_peer_id(); + let transport = MemoryTransport::default() + .upgrade(upgrade::Version::V1) + .authenticate(NoiseConfig::xx(dh_key).into_authenticated()) + .multiplex(yamux::Config::default()) + .map(|(peer, muxer), _| (peer, StreamMuxerBox::new(muxer))) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .boxed(); + Swarm::new(transport, LightClientHandler::new(cf, client, checker, ps), local_peer) + } + + fn make_config() -> super::Config { + super::Config::new(&ProtocolId::from(&b"foo"[..])) + } + + struct EmptyPollParams(PeerId); + + impl PollParameters for EmptyPollParams { + type SupportedProtocolsIter = iter::Empty>; + type ListenedAddressesIter = iter::Empty; + type ExternalAddressesIter = iter::Empty; + + fn supported_protocols(&self) -> Self::SupportedProtocolsIter { + iter::empty() + } + + fn listened_addresses(&self) -> Self::ListenedAddressesIter { + iter::empty() + } + + fn external_addresses(&self) -> Self::ExternalAddressesIter { + iter::empty() + } + + fn local_peer_id(&self) -> &PeerId { + &self.0 + } + } + + fn peerset() -> (sc_peerset::Peerset, sc_peerset::PeersetHandle) { + let cfg = sc_peerset::PeersetConfig { + in_peers: 128, + out_peers: 128, + bootnodes: Vec::new(), + reserved_only: false, + reserved_nodes: Vec::new(), + }; + sc_peerset::Peerset::from_config(cfg) + } + + fn make_behaviour + ( ok: bool + , ps: sc_peerset::PeersetHandle + , cf: super::Config + ) -> LightClientHandler + { + let client = Arc::new(substrate_test_runtime_client::new()); + let checker = Arc::new(DummyFetchChecker::new(ok)); + LightClientHandler::new(cf, client, checker, ps) + } + + fn empty_dialer() -> ConnectedPoint { + ConnectedPoint::Dialer { address: Multiaddr::empty() } + } + + fn poll(mut b: &mut LightClientHandler) -> Poll> { + let mut p = EmptyPollParams(PeerId::random()); + match future::poll_fn(|cx| Pin::new(&mut b).poll(cx, &mut p)).now_or_never() { + Some(a) => Poll::Ready(a), + None => Poll::Pending + } + } + + #[test] + fn disconnects_from_peer_if_told() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + behaviour.inject_disconnected(&peer, empty_dialer()); + assert_eq!(0, behaviour.peers.len()) + } + + #[test] + fn disconnects_from_peer_if_request_times_out() { + let peer0 = PeerId::random(); + let peer1 = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer0.clone(), empty_dialer()); + behaviour.inject_connected(peer1.clone(), empty_dialer()); + + // We now know about two peers. + assert_eq!(HashSet::from_iter(&[peer0.clone(), peer1.clone()]), behaviour.peers.keys().collect::>()); + + // No requests have been made yet. + assert!(behaviour.pending_requests.is_empty()); + assert!(behaviour.outstanding.is_empty()); + + // Issue our first request! + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + assert_eq!(1, behaviour.pending_requests.len()); + + // The behaviour should now attempt to send the request. + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, .. }) => { + assert!(peer_id == peer0 || peer_id == peer1) + }); + + // And we should have one busy peer. + assert!({ + let (idle, busy): (Vec<_>, Vec<_>) = + behaviour.peers.iter().partition(|(_, info)| info.status == PeerStatus::Idle); + + idle.len() == 1 && busy.len() == 1 + && (idle[0].0 == &peer0 || busy[0].0 == &peer0) + && (idle[0].0 == &peer1 || busy[0].0 == &peer1) + }); + + // No more pending requests, but one should be outstanding. + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + // We now set back the timestamp of the outstanding request to make it expire. + let request = behaviour.outstanding.values_mut().next().unwrap(); + request.timestamp -= make_config().request_timeout; + + // Make progress, but do not expect some action. + assert_matches!(poll(&mut behaviour), Poll::Pending); + + // The request should have timed out by now and the corresponding peer be removed. + assert_eq!(1, behaviour.peers.len()); + // Since we asked for one retry, the request should be back in the pending queue. + assert_eq!(1, behaviour.pending_requests.len()); + // No other request should be ongoing. + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_response_with_wrong_id() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + // Construct response with bogus ID + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 2365789, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + // Make sure our bogus ID is really not used. + assert!(!behaviour.outstanding.keys().any(|id| id == &response.id)); + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_incorrect_response() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(false, pset.1, make_config()); + // ^--- Making sure the response data check fails. + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + let request_id = *behaviour.outstanding.keys().next().unwrap(); + + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_unexpected_response() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + + // Some unsolicited response + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 2347895932, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + + assert!(behaviour.peers.is_empty()); + poll(&mut behaviour); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_wrong_response_type() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + let request_id = *behaviour.outstanding.keys().next().unwrap(); + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; // Not a RemoteCallResponse! + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn receives_remote_failure_after_retry_count_failures() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let peer3 = PeerId::random(); + let peer4 = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(false, pset.1, make_config()); + // ^--- Making sure the response data check fails. + + behaviour.inject_connected(peer1.clone(), empty_dialer()); + behaviour.inject_connected(peer2.clone(), empty_dialer()); + behaviour.inject_connected(peer3.clone(), empty_dialer()); + behaviour.inject_connected(peer4.clone(), empty_dialer()); + assert_eq!(4, behaviour.peers.len()); + + let mut chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(3), // Attempt up to three retries. + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + for _ in 0 .. 3 { + // Construct an invalid response + let request_id = *behaviour.outstanding.keys().next().unwrap(); + let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)) + } + }; + behaviour.inject_node_event(responding_peer, Event::Response(response.clone())); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(chan.1.try_recv(), Ok(None)) + } + // Final invalid response + let request_id = *behaviour.outstanding.keys().next().unwrap(); + let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + behaviour.inject_node_event(responding_peer, Event::Response(response)); + assert_matches!(poll(&mut behaviour), Poll::Pending); + assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) + } + + fn issue_request(request: Request) { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let response = match request { + Request::Header{..} => { + let r = api::v1::light::RemoteHeaderResponse { + header: dummy_header().encode(), + proof: empty_proof() + }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteHeaderResponse(r)), + } + } + Request::Read{..} => { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + } + Request::ReadChild{..} => { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + } + Request::Call{..} => { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + } + Request::Changes{..} => { + let r = api::v1::light::RemoteChangesResponse { + max: iter::repeat(1).take(32).collect(), + proof: Vec::new(), + roots: Vec::new(), + roots_proof: empty_proof() + }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteChangesResponse(r)), + } + } + }; + + behaviour.request(request).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + assert_eq!(1, *behaviour.outstanding.keys().next().unwrap()); + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + + poll(&mut behaviour); + + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()) + } + + #[test] + fn receives_remote_call_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }; + issue_request(Request::Call { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_read_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteReadRequest { + header: dummy_header(), + block: Default::default(), + keys: vec![b":key".to_vec()], + retry_count: None, + }; + issue_request(Request::Read { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_read_child_response() { + let info = CHILD_INFO.info(); + let mut chan = oneshot::channel(); + let request = fetcher::RemoteReadChildRequest { + header: dummy_header(), + block: Default::default(), + storage_key: b":child_storage:sub".to_vec(), + keys: vec![b":key".to_vec()], + child_info: info.0.to_vec(), + child_type: info.1, + retry_count: None, + }; + issue_request(Request::ReadChild { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_header_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteHeaderRequest { + cht_root: Default::default(), + block: 1, + retry_count: None, + }; + issue_request(Request::Header { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_changes_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteChangesRequest { + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), + }], + first_block: (1, Default::default()), + last_block: (100, Default::default()), + max_block: (100, Default::default()), + tries_roots: (1, Default::default(), Vec::new()), + key: Vec::new(), + storage_key: None, + retry_count: None, + }; + issue_request(Request::Changes { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + fn send_receive(request: Request) { + // We start a swarm on the listening side which awaits incoming requests and answers them: + let local_pset = peerset(); + let local_listen_addr: libp2p::Multiaddr = libp2p::multiaddr::Protocol::Memory(rand::random()).into(); + let mut local_swarm = make_swarm(true, local_pset.1, make_config()); + Swarm::listen_on(&mut local_swarm, local_listen_addr.clone()).unwrap(); + + // We also start a swarm that makes requests and awaits responses: + let remote_pset = peerset(); + let mut remote_swarm = make_swarm(true, remote_pset.1, make_config()); + + // We now schedule a request, dial the remote and let the two swarm work it out: + remote_swarm.request(request).unwrap(); + Swarm::dial_addr(&mut remote_swarm, local_listen_addr).unwrap(); + + let future = { + let a = local_swarm.for_each(|_| future::ready(())); + let b = remote_swarm.for_each(|_| future::ready(())); + future::join(a, b).map(|_| ()) + }; + + task::spawn(future); + } + + #[test] + fn send_receive_call() { + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }; + send_receive(Request::Call { request, sender: chan.0 }); + assert_eq!(vec![42], task::block_on(chan.1).unwrap().unwrap()); + // ^--- from `DummyFetchChecker::check_execution_proof` + } + + #[test] + fn send_receive_read() { + let chan = oneshot::channel(); + let request = fetcher::RemoteReadRequest { + header: dummy_header(), + block: Default::default(), + keys: vec![b":key".to_vec()], + retry_count: None + }; + send_receive(Request::Read { request, sender: chan.0 }); + assert_eq!(Some(vec![42]), task::block_on(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()); + // ^--- from `DummyFetchChecker::check_read_proof` + } + + #[test] + fn send_receive_read_child() { + let info = CHILD_INFO.info(); + let chan = oneshot::channel(); + let request = fetcher::RemoteReadChildRequest { + header: dummy_header(), + block: Default::default(), + storage_key: b":child_storage:sub".to_vec(), + keys: vec![b":key".to_vec()], + child_info: info.0.to_vec(), + child_type: info.1, + retry_count: None, + }; + send_receive(Request::ReadChild { request, sender: chan.0 }); + assert_eq!(Some(vec![42]), task::block_on(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()); + // ^--- from `DummyFetchChecker::check_read_child_proof` + } + + #[test] + fn send_receive_header() { + let _ = env_logger::try_init(); + let chan = oneshot::channel(); + let request = fetcher::RemoteHeaderRequest { + cht_root: Default::default(), + block: 1, + retry_count: None, + }; + send_receive(Request::Header { request, sender: chan.0 }); + // The remote does not know block 1: + assert_matches!(task::block_on(chan.1).unwrap(), Err(ClientError::RemoteFetchFailed)); + } + + #[test] + fn send_receive_changes() { + let chan = oneshot::channel(); + let request = fetcher::RemoteChangesRequest { + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), + }], + first_block: (1, Default::default()), + last_block: (100, Default::default()), + max_block: (100, Default::default()), + tries_roots: (1, Default::default(), Vec::new()), + key: Vec::new(), + storage_key: None, + retry_count: None, + }; + send_receive(Request::Changes { request, sender: chan.0 }); + assert_eq!(vec![(100, 2)], task::block_on(chan.1).unwrap().unwrap()); + // ^--- from `DummyFetchChecker::check_changes_proof` + } +} diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index bfa8daa181ca15f175a5c11fb2bf531da86f719a..aff220b6e03a6cdf41389d881e62f73497cea003 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -21,7 +21,8 @@ use std::collections::{HashMap, VecDeque}; use std::sync::Arc; -use std::time::{Instant, Duration}; +use std::time::Duration; +use wasm_timer::Instant; use log::{trace, info}; use futures::channel::oneshot::{Sender as OneShotSender}; use linked_hash_map::{Entry, LinkedHashMap}; @@ -29,7 +30,7 @@ use sp_blockchain::Error as ClientError; use sc_client_api::{FetchChecker, RemoteHeaderRequest, RemoteCallRequest, RemoteReadRequest, RemoteChangesRequest, ChangesProof, RemoteReadChildRequest, RemoteBodyRequest, StorageProof}; -use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; +use crate::protocol::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; use libp2p::PeerId; use crate::config::Roles; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; @@ -40,7 +41,7 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); /// Default request retry count. const RETRY_COUNT: usize = 1; /// Reputation change for a peer when a request timed out. -const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); +pub(crate) const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); /// Trait used by the `LightDispatch` service to communicate messages back to the network. pub trait LightDispatchNetwork { @@ -226,7 +227,7 @@ impl FetchChecker for AlwaysBadChecker { impl LightDispatch where B::Header: HeaderT, { - /// Creates new light client requests processer. + /// Creates new light client requests processor. pub fn new(checker: Arc>) -> Self { LightDispatch { checker, @@ -503,7 +504,7 @@ impl LightDispatch where } pub fn is_light_response(&self, peer: &PeerId, request_id: message::RequestId) -> bool { - self.active_peers.get(&peer).map_or(false, |r| r.id == request_id) + self.active_peers.get(peer).map_or(false, |r| r.id == request_id) } fn remove(&mut self, peer: PeerId, id: u64) -> Option> { @@ -566,7 +567,7 @@ impl LightDispatch where // return peer to the back of the queue self.idle_peers.push_back(peer.clone()); - // we have enumerated all peers and noone can handle request + // we have enumerated all peers and no one can handle request if Some(peer) == last_peer { let request = self.pending_requests.pop_front().expect("checked in loop condition; qed"); unhandled_requests.push_back(request); @@ -691,17 +692,26 @@ pub mod tests { use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; use libp2p::PeerId; use super::{REQUEST_TIMEOUT, LightDispatch, LightDispatchNetwork, RequestData, StorageProof}; - use sp_test_primitives::{Block, Extrinsic, Header}; + use sp_test_primitives::{Block, Header}; - struct DummyFetchChecker { ok: bool } + pub(crate) struct DummyFetchChecker { + pub(crate) ok: bool, + _mark: std::marker::PhantomData + } + + impl DummyFetchChecker { + pub(crate) fn new(ok: bool) -> Self { + DummyFetchChecker { ok, _mark: std::marker::PhantomData } + } + } - impl FetchChecker for DummyFetchChecker { + impl FetchChecker for DummyFetchChecker { fn check_header_proof( &self, - _request: &RemoteHeaderRequest
, - header: Option
, + _request: &RemoteHeaderRequest, + header: Option, _remote_proof: StorageProof, - ) -> ClientResult
{ + ) -> ClientResult { match self.ok { true if header.is_some() => Ok(header.unwrap()), _ => Err(ClientError::Backend("Test error".into())), @@ -710,7 +720,7 @@ pub mod tests { fn check_read_proof( &self, - request: &RemoteReadRequest
, + request: &RemoteReadRequest, _: StorageProof, ) -> ClientResult, Option>>> { match self.ok { @@ -726,7 +736,7 @@ pub mod tests { fn check_read_child_proof( &self, - request: &RemoteReadChildRequest
, + request: &RemoteReadChildRequest, _: StorageProof, ) -> ClientResult, Option>>> { match self.ok { @@ -740,7 +750,7 @@ pub mod tests { } } - fn check_execution_proof(&self, _: &RemoteCallRequest
, _: StorageProof) -> ClientResult> { + fn check_execution_proof(&self, _: &RemoteCallRequest, _: StorageProof) -> ClientResult> { match self.ok { true => Ok(vec![42]), false => Err(ClientError::Backend("Test error".into())), @@ -749,20 +759,20 @@ pub mod tests { fn check_changes_proof( &self, - _: &RemoteChangesRequest
, - _: ChangesProof
- ) -> ClientResult, u32)>> { + _: &RemoteChangesRequest, + _: ChangesProof + ) -> ClientResult, u32)>> { match self.ok { - true => Ok(vec![(100, 2)]), + true => Ok(vec![(100.into(), 2)]), false => Err(ClientError::Backend("Test error".into())), } } fn check_body_proof( &self, - _: &RemoteBodyRequest
, - body: Vec - ) -> ClientResult> { + _: &RemoteBodyRequest, + body: Vec + ) -> ClientResult> { match self.ok { true => Ok(body), false => Err(ClientError::Backend("Test error".into())), @@ -771,7 +781,7 @@ pub mod tests { } fn dummy(ok: bool) -> LightDispatch { - LightDispatch::new(Arc::new(DummyFetchChecker { ok })) + LightDispatch::new(Arc::new(DummyFetchChecker::new(ok))) } fn total_peers(light_dispatch: &LightDispatch) -> usize { @@ -790,7 +800,7 @@ pub mod tests { }); } - fn dummy_header() -> Header { + pub(crate) fn dummy_header() -> Header { Header { parent_hash: Default::default(), number: 0, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index ef7d550de6cbeac8f0250aae5ebe419d04209ab9..a12c26da2e47eec14ae136ede05e6094d9e0c715 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -209,19 +209,16 @@ pub mod generic { RemoteHeaderResponse(RemoteHeaderResponse
), /// Remote changes request. RemoteChangesRequest(RemoteChangesRequest), - /// Remote changes reponse. + /// Remote changes response. RemoteChangesResponse(RemoteChangesResponse), /// Remote child storage read request. RemoteReadChildRequest(RemoteReadChildRequest), /// Finality proof request. FinalityProofRequest(FinalityProofRequest), - /// Finality proof reponse. + /// Finality proof response. FinalityProofResponse(FinalityProofResponse), /// Batch of consensus protocol messages. ConsensusBatch(Vec), - /// Chain-specific message. - #[codec(index = "255")] - ChainSpecific(Vec), } impl Message { @@ -246,7 +243,6 @@ pub mod generic { Message::FinalityProofRequest(_) => "FinalityProofRequest", Message::FinalityProofResponse(_) => "FinalityProofResponse", Message::ConsensusBatch(_) => "ConsensusBatch", - Message::ChainSpecific(_) => "ChainSpecific", } } } diff --git a/client/network/src/protocol/schema/api.v1.proto b/client/network/src/protocol/schema/api.v1.proto new file mode 100644 index 0000000000000000000000000000000000000000..73128c53de46635a1af74cd78c6f19087028af7a --- /dev/null +++ b/client/network/src/protocol/schema/api.v1.proto @@ -0,0 +1,59 @@ +// Schema definition for block request/response messages. + +syntax = "proto3"; + +package api.v1; + +// Block enumeration direction. +enum Direction { + // Enumerate in ascending order (from child to parent). + Ascending = 0; + // Enumerate in descending order (from parent to canonical child). + Descending = 1; +} + +// Request block data from a peer. +message BlockRequest { + // Unique request id. + uint64 id = 1; + // Bits of block data to request. + uint32 fields = 2; + // Start from this block. + oneof from_block { + // Start with given hash. + bytes hash = 3; + // Start with given block number. + bytes number = 4; + } + // End at this block. An implementation defined maximum is used when unspecified. + bytes to_block = 5; // optional + // Sequence direction. + Direction direction = 6; + // Maximum number of blocks to return. An implementation defined maximum is used when unspecified. + uint32 max_blocks = 7; // optional +} + +// Response to `BlockRequest` +message BlockResponse { + // Id of a request this response was made for. + uint64 id = 1; + // Block data for the requested sequence. + repeated BlockData blocks = 2; +} + +// Block data sent in the response. +message BlockData { + // Block header hash. + bytes hash = 1; + // Block header if requested. + bytes header = 2; // optional + // Block body if requested. + repeated bytes body = 3; // optional + // Block receipt if requested. + bytes receipt = 4; // optional + // Block message queue if requested. + bytes message_queue = 5; // optional + // Justification if requested. + bytes justification = 6; // optional +} + diff --git a/client/network/src/protocol/schema/light.v1.proto b/client/network/src/protocol/schema/light.v1.proto new file mode 100644 index 0000000000000000000000000000000000000000..b9aee67b5ee24ff6614e3db2a0c711988a8d03c1 --- /dev/null +++ b/client/network/src/protocol/schema/light.v1.proto @@ -0,0 +1,128 @@ +// Schema definition for light client messages. + +syntax = "proto3"; + +package api.v1.light; + +// A pair of arbitrary bytes. +message Pair { + // The first element of the pair. + bytes fst = 1; + // The second element of the pair. + bytes snd = 2; +} + +// Enumerate all possible light client request messages. +message Request { + // Unique request id. + uint64 id = 1; + oneof request { + RemoteCallRequest remote_call_request = 2; + RemoteReadRequest remote_read_request = 3; + RemoteHeaderRequest remote_header_request = 4; + RemoteReadChildRequest remote_read_child_request = 5; + RemoteChangesRequest remote_changes_request = 6; + } +} + +// Enumerate all possible light client response messages. +message Response { + /// Id of a request this response was made for. + uint64 id = 1; + oneof response { + RemoteCallResponse remote_call_response = 2; + RemoteReadResponse remote_read_response = 3; + RemoteHeaderResponse remote_header_response = 4; + RemoteChangesResponse remote_changes_response = 6; + } +} + +// Remote call request. +message RemoteCallRequest { + // Block at which to perform call. + bytes block = 2; + // Method name. + string method = 3; + // Call data. + bytes data = 4; +} + +// Remote call response. +message RemoteCallResponse { + // Execution proof. + bytes proof = 2; +} + +// Remote storage read request. +message RemoteReadRequest { + // Block at which to perform call. + bytes block = 2; + // Storage keys. + repeated bytes keys = 3; +} + +// Remote read response. +message RemoteReadResponse { + // Read proof. + bytes proof = 2; +} + +// Remote storage read child request. +message RemoteReadChildRequest { + // Block at which to perform call. + bytes block = 2; + // Child Storage key. + bytes storage_key = 3; + // Child trie source information. + bytes child_info = 4; + /// Child type, its required to resolve `child_info` + /// content and choose child implementation. + uint32 child_type = 5; + // Storage keys. + repeated bytes keys = 6; +} + +// Remote header request. +message RemoteHeaderRequest { + // Block number to request header for. + bytes block = 2; +} + +// Remote header response. +message RemoteHeaderResponse { + // Header. None if proof generation has failed (e.g. header is unknown). + bytes header = 2; // optional + // Header proof. + bytes proof = 3; +} + +/// Remote changes request. +message RemoteChangesRequest { + // Hash of the first block of the range (including first) where changes are requested. + bytes first = 2; + // Hash of the last block of the range (including last) where changes are requested. + bytes last = 3; + // Hash of the first block for which the requester has the changes trie root. All other + // affected roots must be proved. + bytes min = 4; + // Hash of the last block that we can use when querying changes. + bytes max = 5; + // Storage child node key which changes are requested. + bytes storage_key = 6; // optional + // Storage key which changes are requested. + bytes key = 7; +} + +// Remote changes response. +message RemoteChangesResponse { + // Proof has been generated using block with this number as a max block. Should be + // less than or equal to the RemoteChangesRequest::max block number. + bytes max = 2; + // Changes proof. + repeated bytes proof = 3; + // Changes tries roots missing on the requester' node. + repeated Pair roots = 4; + // Missing changes tries roots proof. + bytes roots_proof = 5; +} + diff --git a/client/network/src/protocol/specialization.rs b/client/network/src/protocol/specialization.rs deleted file mode 100644 index af6d5f7a239cd8f384d53955e3efbfb3372f28eb..0000000000000000000000000000000000000000 --- a/client/network/src/protocol/specialization.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Specializations of the substrate network protocol to allow more complex forms of communication. - -pub use crate::protocol::event::{DhtEvent, Event}; - -use crate::protocol::Context; -use libp2p::PeerId; -use sp_runtime::traits::Block as BlockT; - -/// A specialization of the substrate network protocol. Handles events and sends messages. -pub trait NetworkSpecialization: Send + Sync + 'static { - /// Get the current specialization-status. - fn status(&self) -> Vec; - - /// Called when a peer successfully handshakes. - fn on_connect(&mut self, ctx: &mut dyn Context, who: PeerId, status: crate::message::Status); - - /// Called when a peer is disconnected. If the peer ID is unknown, it should be ignored. - fn on_disconnect(&mut self, ctx: &mut dyn Context, who: PeerId); - - /// Called when a network-specific message arrives. - fn on_message( - &mut self, - ctx: &mut dyn Context, - who: PeerId, - message: Vec - ); - - /// Called when a network-specific event arrives. - #[deprecated(note = "This method is never called; please use `with_dht_event_tx` when building the service")] - fn on_event(&mut self, _event: Event) {} - - /// Called on abort. - #[deprecated(note = "This method is never called; aborting corresponds to dropping the object")] - fn on_abort(&mut self) { } - - /// Called periodically to maintain peers and handle timeouts. - fn maintain_peers(&mut self, _ctx: &mut dyn Context) { } - - /// Called when a block is _imported_ at the head of the chain (not during major sync). - /// Not guaranteed to be called for every block, but will be most of the after major sync. - fn on_block_imported(&mut self, _ctx: &mut dyn Context, _hash: B::Hash, _header: &B::Header) { } -} - -/// A specialization that does nothing. -#[derive(Clone)] -pub struct DummySpecialization; - -impl NetworkSpecialization for DummySpecialization { - fn status(&self) -> Vec { - vec![] - } - - fn on_connect( - &mut self, - _ctx: &mut dyn Context, - _peer_id: PeerId, - _status: crate::message::Status - ) {} - - fn on_disconnect(&mut self, _ctx: &mut dyn Context, _peer_id: PeerId) {} - - fn on_message( - &mut self, - _ctx: &mut dyn Context, - _peer_id: PeerId, - _message: Vec, - ) {} -} - -/// Construct a simple protocol that is composed of several sub protocols. -/// Each "sub protocol" needs to implement `Specialization` and needs to provide a `new()` function. -/// For more fine grained implementations, this macro is not usable. -/// -/// # Example -/// -/// ```nocompile -/// construct_simple_protocol! { -/// pub struct MyProtocol where Block = MyBlock { -/// consensus_gossip: ConsensusGossip, -/// other_protocol: MyCoolStuff, -/// } -/// } -/// ``` -/// -/// You can also provide an optional parameter after `where Block = MyBlock`, so it looks like -/// `where Block = MyBlock, Status = consensus_gossip`. This will instruct the implementation to -/// use the `status()` function from the `ConsensusGossip` protocol. By default, `status()` returns -/// an empty vector. -#[macro_export] -macro_rules! construct_simple_protocol { - ( - $( #[ $attr:meta ] )* - pub struct $protocol:ident where - Block = $block:ident - $( , Status = $status_protocol_name:ident )* - { - $( $sub_protocol_name:ident : $sub_protocol:ident $( <$protocol_block:ty> )*, )* - } - ) => { - $( #[$attr] )* - pub struct $protocol { - $( $sub_protocol_name: $sub_protocol $( <$protocol_block> )*, )* - } - - impl $protocol { - /// Instantiate a node protocol handler. - pub fn new() -> Self { - Self { - $( $sub_protocol_name: $sub_protocol::new(), )* - } - } - } - - impl $crate::specialization::NetworkSpecialization<$block> for $protocol { - fn status(&self) -> Vec { - $( - let status = self.$status_protocol_name.status(); - - if !status.is_empty() { - return status; - } - )* - - Vec::new() - } - - fn on_connect( - &mut self, - _ctx: &mut $crate::Context<$block>, - _who: $crate::PeerId, - _status: $crate::StatusMessage<$block> - ) { - $( self.$sub_protocol_name.on_connect(_ctx, _who, _status); )* - } - - fn on_disconnect(&mut self, _ctx: &mut $crate::Context<$block>, _who: $crate::PeerId) { - $( self.$sub_protocol_name.on_disconnect(_ctx, _who); )* - } - - fn on_message( - &mut self, - _ctx: &mut $crate::Context<$block>, - _who: $crate::PeerId, - _message: Vec, - ) { - $( self.$sub_protocol_name.on_message(_ctx, _who, _message); )* - } - - fn on_event( - &mut self, - _event: $crate::specialization::Event - ) { - $( self.$sub_protocol_name.on_event(_event); )* - } - - fn on_abort(&mut self) { - $( self.$sub_protocol_name.on_abort(); )* - } - - fn maintain_peers(&mut self, _ctx: &mut $crate::Context<$block>) { - $( self.$sub_protocol_name.maintain_peers(_ctx); )* - } - - fn on_block_imported( - &mut self, - _ctx: &mut $crate::Context<$block>, - _hash: <$block as $crate::BlockT>::Hash, - _header: &<$block as $crate::BlockT>::Header - ) { - $( self.$sub_protocol_name.on_block_imported(_ctx, _hash, _header); )* - } - } - } -} diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index a513d47ca419862db6598ba4e421a890b0ff3864..b1cd89155effed3647c9ce445914ad96fc88b380 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -35,7 +35,7 @@ use sp_consensus::{BlockOrigin, BlockStatus, }; use crate::{ config::{Roles, BoxFinalityProofRequestBuilder}, - message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, + protocol::message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, FinalityProofResponse}, }; use either::Either; @@ -751,7 +751,7 @@ impl ChainSync { | PeerSyncState::DownloadingFinalityProof(..) => Vec::new() } } else { - // When request.is_none() just acccept blocks + // When request.is_none() this is a block announcement. Just accept blocks. blocks.into_iter().map(|b| { IncomingBlock { hash: b.hash, @@ -813,7 +813,7 @@ impl ChainSync { peer.state = PeerSyncState::Available; // We only request one justification at a time - if let Some(block) = response.blocks.into_iter().next() { + let justification = if let Some(block) = response.blocks.into_iter().next() { if hash != block.hash { info!( target: "sync", @@ -821,13 +821,22 @@ impl ChainSync { ); return Err(BadPeer(who, rep::BAD_JUSTIFICATION)); } - if let Some((peer, hash, number, j)) = self.extra_justifications.on_response(who, block.justification) { - return Ok(OnBlockJustification::Import { peer, hash, number, justification: j }) - } + + block.justification } else { - // we might have asked the peer for a justification on a block that we thought it had - // (regardless of whether it had a justification for it or not). - trace!(target: "sync", "Peer {:?} provided empty response for justification request {:?}", who, hash) + // we might have asked the peer for a justification on a block that we assumed it + // had but didn't (regardless of whether it had a justification for it or not). + trace!(target: "sync", + "Peer {:?} provided empty response for justification request {:?}", + who, + hash, + ); + + None + }; + + if let Some((peer, hash, number, j)) = self.extra_justifications.on_response(who, justification) { + return Ok(OnBlockJustification::Import { peer, hash, number, justification: j }) } } @@ -955,7 +964,7 @@ impl ChainSync { }, Err(BlockImportError::MissingState) => { // This may happen if the chain we were requesting upon has been discarded - // in the meantime becasue other chain has been finalized. + // in the meantime because other chain has been finalized. // Don't mark it as bad as it still may be synced if explicitly requested. trace!(target: "sync", "Obsolete block"); }, @@ -1353,3 +1362,93 @@ fn fork_sync_request( } None } + +#[cfg(test)] +mod test { + use super::*; + use super::message::FromBlock; + use substrate_test_runtime_client::{ + runtime::Block, + DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, + }; + use sp_blockchain::HeaderBackend; + use sp_consensus::block_validation::DefaultBlockAnnounceValidator; + + #[test] + fn processes_empty_response_on_justification_request_for_unknown_block() { + // if we ask for a justification for a given block to a peer that doesn't know that block + // (different from not having a justification), the peer will reply with an empty response. + // internally we should process the response as the justification not being available. + + let client = Arc::new(TestClientBuilder::new().build()); + let info = client.info(); + let block_announce_validator = Box::new(DefaultBlockAnnounceValidator::new(client.clone())); + let peer_id = PeerId::random(); + + let mut sync = ChainSync::new( + Roles::AUTHORITY, + client.clone(), + &info, + None, + block_announce_validator, + 1, + ); + + let (a1_hash, a1_number) = { + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + (a1.hash(), *a1.header.number()) + }; + + // add a new peer with the same best block + sync.new_peer(peer_id.clone(), a1_hash, a1_number).unwrap(); + + // and request a justification for the block + sync.request_justification(&a1_hash, a1_number); + + // the justification request should be scheduled to that peer + assert!( + sync.justification_requests().any(|(who, request)| { + who == peer_id && request.from == FromBlock::Hash(a1_hash) + }) + ); + + // there are no extra pending requests + assert_eq!( + sync.extra_justifications.pending_requests().count(), + 0, + ); + + // there's one in-flight extra request to the expected peer + assert!( + sync.extra_justifications.active_requests().any(|(who, (hash, number))| { + *who == peer_id && *hash == a1_hash && *number == a1_number + }) + ); + + // if the peer replies with an empty response (i.e. it doesn't know the block), + // the active request should be cleared. + assert_eq!( + sync.on_block_justification( + peer_id.clone(), + BlockResponse:: { + id: 0, + blocks: vec![], + } + ), + Ok(OnBlockJustification::Nothing), + ); + + // there should be no in-flight requests + assert_eq!( + sync.extra_justifications.active_requests().count(), + 0, + ); + + // and the request should now be pending again, waiting for reschedule + assert!( + sync.extra_justifications.pending_requests().any(|(hash, number)| { + *hash == a1_hash && *number == a1_number + }) + ); + } +} diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index 974935f765e6c09a9c390b2b5aa17e5813bb9fed..d4e4581c670bd1f2f1abf2a9c00302aed68b44d1 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -22,7 +22,7 @@ use std::collections::hash_map::Entry; use log::trace; use libp2p::PeerId; use sp_runtime::traits::{Block as BlockT, NumberFor, One}; -use crate::message; +use crate::protocol::message; /// Block data with origin. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 44b42b154fff2724d78abf36b0d6d5247023f49d..38c250cddf26d920d9e2f2e2e08eed19ac53a3e4 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -21,7 +21,8 @@ use libp2p::PeerId; use log::{debug, trace, warn}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use std::collections::{HashMap, HashSet, VecDeque}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; // Time to wait before trying to get the same extra data from the same peer. const EXTRA_RETRY_WAIT: Duration = Duration::from_secs(10); @@ -227,6 +228,18 @@ impl ExtraRequests { true } + + /// Returns an iterator over all active (in-flight) requests and associated peer id. + #[cfg(test)] + pub(crate) fn active_requests(&self) -> impl Iterator)> { + self.active_requests.iter() + } + + /// Returns an iterator over all scheduled pending requests. + #[cfg(test)] + pub(crate) fn pending_requests(&self) -> impl Iterator> { + self.pending_requests.iter() + } } /// Matches peers with pending extra requests. @@ -468,7 +481,7 @@ mod tests { } #[test] - fn anecstor_roots_are_finalized_when_finality_notification_is_missed() { + fn ancestor_roots_are_finalized_when_finality_notification_is_missed() { let mut finality_proofs = ExtraRequests::::new("test"); let hash4 = [4; 32].into(); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index e3eaf6c31d5665ab387329323648cd66d1abc0a5..288289d95c81ed64197115a42ce5542b88b16839 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -25,7 +25,7 @@ //! The methods of the [`NetworkService`] are implemented by sending a message over a channel, //! which is then processed by [`NetworkWorker::poll`]. -use std::{collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path}; +use std::{borrow::Cow, collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path}; use std::sync::{Arc, atomic::{AtomicBool, AtomicUsize, Ordering}}; use std::pin::Pin; use std::task::Poll; @@ -35,20 +35,18 @@ use sp_consensus::import_queue::{BlockImportResult, BlockImportError}; use futures::{prelude::*, channel::mpsc}; use log::{warn, error, info, trace}; use libp2p::{PeerId, Multiaddr, kad::record}; -use libp2p::core::{transport::boxed::Boxed, muxing::StreamMuxerBox}; use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent}; use parking_lot::Mutex; use sc_peerset::PeersetHandle; use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId}; use crate::{behaviour::{Behaviour, BehaviourOut}, config::{parse_str_addr, parse_addr}}; -use crate::{NetworkState, NetworkStateNotConnectedPeer, NetworkStatePeer}; use crate::{transport, config::NonReservedPeerMode, ReputationChange}; use crate::config::{Params, TransportConfig}; use crate::error::Error; -use crate::protocol::{self, Protocol, Context, PeerInfo}; +use crate::network_state::{NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer}; +use crate::protocol::{self, Protocol, PeerInfo}; use crate::protocol::{event::Event, light_dispatch::{AlwaysBadChecker, RequestData}}; -use crate::protocol::specialization::NetworkSpecialization; use crate::protocol::sync::SyncState; /// Minimum Requirements for a Hash within Networking @@ -102,7 +100,7 @@ impl ReportHandle { } /// Substrate network service. Handles network IO and manages connectivity. -pub struct NetworkService, H: ExHashT> { +pub struct NetworkService { /// Number of peers we're connected to. num_connected: Arc, /// The local external addresses. @@ -117,19 +115,19 @@ pub struct NetworkService, H: E /// nodes it should be connected to or not. peerset: PeersetHandle, /// Channel that sends messages to the actual worker. - to_worker: mpsc::UnboundedSender>, + to_worker: mpsc::UnboundedSender>, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, } -impl, H: ExHashT> NetworkWorker { +impl NetworkWorker { /// Creates the network service. /// /// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. - pub fn new(params: Params) -> Result, Error> { + pub fn new(params: Params) -> Result, Error> { let (to_worker, from_worker) = mpsc::unbounded(); if let Some(ref path) = params.network_config.net_config_path { @@ -193,6 +191,10 @@ impl, H: ExHashT> NetworkWorker let local_peer_id = local_public.clone().into_peer_id(); info!(target: "sub-libp2p", "Local node identity is: {}", local_peer_id.to_base58()); + let checker = params.on_demand.as_ref() + .map(|od| od.checker().clone()) + .unwrap_or(Arc::new(AlwaysBadChecker)); + let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); let (protocol, peerset_handle) = Protocol::new( @@ -200,25 +202,31 @@ impl, H: ExHashT> NetworkWorker roles: params.roles, max_parallel_downloads: params.network_config.max_parallel_downloads, }, - params.chain, - params.on_demand.as_ref().map(|od| od.checker().clone()) - .unwrap_or(Arc::new(AlwaysBadChecker)), - params.specialization, + params.chain.clone(), + checker.clone(), params.transaction_pool, - params.finality_proof_provider, + params.finality_proof_provider.clone(), params.finality_proof_request_builder, - params.protocol_id, + params.protocol_id.clone(), peerset_config, params.block_announce_validator )?; // Build the swarm. - let (mut swarm, bandwidth): (Swarm::, _) = { + let (mut swarm, bandwidth): (Swarm::, _) = { let user_agent = format!( "{} ({})", params.network_config.client_version, params.network_config.node_name ); + let block_requests = { + let config = protocol::block_requests::Config::new(¶ms.protocol_id); + protocol::BlockRequests::new(config, params.chain.clone()) + }; + let light_client_handler = { + let config = protocol::light_client_handler::Config::new(¶ms.protocol_id); + protocol::LightClientHandler::new(config, params.chain, checker, peerset_handle.clone()) + }; let behaviour = futures::executor::block_on(Behaviour::new( protocol, user_agent, @@ -232,14 +240,17 @@ impl, H: ExHashT> NetworkWorker TransportConfig::MemoryOnly => false, TransportConfig::Normal { allow_private_ipv4, .. } => allow_private_ipv4, }, + u64::from(params.network_config.out_peers) + 15, + block_requests, + light_client_handler )); let (transport, bandwidth) = { - let (config_mem, config_wasm) = match params.network_config.transport { - TransportConfig::MemoryOnly => (true, None), - TransportConfig::Normal { wasm_external_transport, .. } => - (false, wasm_external_transport) + let (config_mem, config_wasm, flowctrl) = match params.network_config.transport { + TransportConfig::MemoryOnly => (true, None, false), + TransportConfig::Normal { wasm_external_transport, use_yamux_flow_control, .. } => + (false, wasm_external_transport, use_yamux_flow_control) }; - transport::build_transport(local_identity, config_mem, config_wasm) + transport::build_transport(local_identity, config_mem, config_wasm, flowctrl) }; let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()); if let Some(spawner) = params.executor { @@ -250,14 +261,14 @@ impl, H: ExHashT> NetworkWorker // Listen on multiaddresses. for addr in ¶ms.network_config.listen_addresses { - if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { + if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { warn!(target: "sub-libp2p", "Can't listen on {} because: {:?}", addr, err) } } // Add external addresses. for addr in ¶ms.network_config.public_addresses { - Swarm::::add_external_address(&mut swarm, addr.clone()); + Swarm::::add_external_address(&mut swarm, addr.clone()); } let external_addresses = Arc::new(Mutex::new(Vec::new())); @@ -338,13 +349,13 @@ impl, H: ExHashT> NetworkWorker /// Return a `NetworkService` that can be shared through the code base and can be used to /// manipulate the worker. - pub fn service(&self) -> &Arc> { + pub fn service(&self) -> &Arc> { &self.service } /// You must call this when a new block is imported by the client. - pub fn on_block_imported(&mut self, hash: B::Hash, header: B::Header, data: Vec, is_best: bool) { - self.network_service.user_protocol_mut().on_block_imported(hash, &header, data, is_best); + pub fn on_block_imported(&mut self, header: B::Header, data: Vec, is_best: bool) { + self.network_service.user_protocol_mut().on_block_imported(&header, data, is_best); } /// You must call this when a new block is finalized by the client. @@ -354,7 +365,7 @@ impl, H: ExHashT> NetworkWorker /// Get network state. /// - /// **Note**: Use this only for debugging. This API is unstable. There are warnings literaly + /// **Note**: Use this only for debugging. This API is unstable. There are warnings literally /// everywhere about this. Please don't use this function to retrieve actual information. pub fn network_state(&mut self) -> NetworkState { let swarm = &mut self.network_service; @@ -402,9 +413,9 @@ impl, H: ExHashT> NetworkWorker }; NetworkState { - peer_id: Swarm::::local_peer_id(&swarm).to_base58(), - listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), - external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), + peer_id: Swarm::::local_peer_id(&swarm).to_base58(), + listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), + external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), average_download_per_sec: self.service.bandwidth.average_download_per_sec(), average_upload_per_sec: self.service.bandwidth.average_upload_per_sec(), connected_peers, @@ -433,7 +444,7 @@ impl, H: ExHashT> NetworkWorker } } -impl, H: ExHashT> NetworkService { +impl NetworkService { /// Writes a message on an open notifications channel. Has no effect if the notifications /// channel with this protocol name is closed. /// @@ -477,9 +488,11 @@ impl, H: ExHashT> NetworkServic pub fn register_notifications_protocol( &self, engine_id: ConsensusEngineId, + protocol_name: impl Into>, ) { let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, + protocol_name: protocol_name.into(), }); } @@ -530,15 +543,6 @@ impl, H: ExHashT> NetworkServic .unbounded_send(ServiceToWorkerMsg::RequestJustification(hash.clone(), number)); } - /// Execute a closure with the chain-specific network specialization. - pub fn with_spec(&self, f: F) - where F: FnOnce(&mut S, &mut dyn Context) + Send + 'static - { - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::ExecuteWithSpec(Box::new(f))); - } - /// Are we in the process of downloading the chain? pub fn is_major_syncing(&self) -> bool { self.is_major_syncing.load(Ordering::Relaxed) @@ -626,8 +630,8 @@ impl, H: ExHashT> NetworkServic } } -impl, H: ExHashT> sp_consensus::SyncOracle - for NetworkService +impl sp_consensus::SyncOracle + for NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -638,8 +642,8 @@ impl, H: ExHashT> sp_consensus: } } -impl<'a, B: BlockT + 'static, S: NetworkSpecialization, H: ExHashT> sp_consensus::SyncOracle - for &'a NetworkService +impl<'a, B: BlockT + 'static, H: ExHashT> sp_consensus::SyncOracle + for &'a NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -659,10 +663,9 @@ pub trait NetworkStateInfo { fn local_peer_id(&self) -> PeerId; } -impl NetworkStateInfo for NetworkService +impl NetworkStateInfo for NetworkService where B: sp_runtime::traits::Block, - S: NetworkSpecialization, H: ExHashT, { /// Returns the local external addresses. @@ -679,12 +682,11 @@ impl NetworkStateInfo for NetworkService /// Messages sent from the `NetworkService` to the `NetworkWorker`. /// /// Each entry corresponds to a method of `NetworkService`. -enum ServiceToWorkerMsg> { +enum ServiceToWorkerMsg { PropagateExtrinsic(H), PropagateExtrinsics, RequestJustification(B::Hash, NumberFor), AnnounceBlock(B::Hash, Vec), - ExecuteWithSpec(Box) + Send>), GetValue(record::Key), PutValue(record::Key, Vec), AddKnownAddress(PeerId, Multiaddr), @@ -697,6 +699,7 @@ enum ServiceToWorkerMsg> { }, RegisterNotifProtocol { engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, }, DisconnectPeer(PeerId), } @@ -705,7 +708,7 @@ enum ServiceToWorkerMsg> { /// /// You are encouraged to poll this in a separate background thread or task. #[must_use = "The NetworkWorker must be polled in order for the network to work"] -pub struct NetworkWorker, H: ExHashT> { +pub struct NetworkWorker { /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. external_addresses: Arc>>, /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. @@ -713,20 +716,20 @@ pub struct NetworkWorker, H: Ex /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. is_major_syncing: Arc, /// The network service that can be extracted and shared through the codebase. - service: Arc>, + service: Arc>, /// The *actual* network. - network_service: Swarm, + network_service: Swarm, /// The import queue that was passed as initialization. import_queue: Box>, /// Messages from the `NetworkService` and that must be processed. - from_worker: mpsc::UnboundedReceiver>, + from_worker: mpsc::UnboundedReceiver>, /// Receiver for queries from the light client that must be processed. light_client_rqs: Option>>, /// Senders for events that happen on the network. event_streams: Vec>, } -impl, H: ExHashT> Future for NetworkWorker { +impl Future for NetworkWorker { type Output = Result<(), io::Error>; fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context) -> Poll { @@ -753,11 +756,6 @@ impl, H: ExHashT> Future for Ne }; match msg { - ServiceToWorkerMsg::ExecuteWithSpec(task) => { - let protocol = this.network_service.user_protocol_mut(); - let (mut context, spec) = protocol.specialization_lock(); - task(spec, &mut context); - }, ServiceToWorkerMsg::AnnounceBlock(hash, data) => this.network_service.user_protocol_mut().announce_block(hash, data), ServiceToWorkerMsg::RequestJustification(hash, number) => @@ -778,8 +776,8 @@ impl, H: ExHashT> Future for Ne this.event_streams.push(sender), ServiceToWorkerMsg::WriteNotification { message, engine_id, target } => this.network_service.user_protocol_mut().write_notification(target, engine_id, message), - ServiceToWorkerMsg::RegisterNotifProtocol { engine_id } => { - let events = this.network_service.user_protocol_mut().register_notifications_protocol(engine_id); + ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => { + let events = this.network_service.user_protocol_mut().register_notifications_protocol(engine_id, protocol_name); for event in events { this.event_streams.retain(|sender| sender.unbounded_send(event.clone()).is_ok()); } @@ -823,7 +821,7 @@ impl, H: ExHashT> Future for Ne // Update the variables shared with the `NetworkService`. this.num_connected.store(this.network_service.user_protocol_mut().num_connected_peers(), Ordering::Relaxed); { - let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); + let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); *this.external_addresses.lock() = external_addresses; } this.is_major_syncing.store(match this.network_service.user_protocol_mut().sync_state() { @@ -835,21 +833,18 @@ impl, H: ExHashT> Future for Ne } } -impl, H: ExHashT> Unpin for NetworkWorker { +impl Unpin for NetworkWorker { } /// The libp2p swarm, customized for our needs. -type Swarm = libp2p::swarm::Swarm< - Boxed<(PeerId, StreamMuxerBox), io::Error>, - Behaviour ->; +type Swarm = libp2p::swarm::Swarm>; // Implementation of `import_queue::Link` trait using the available local variables. -struct NetworkLink<'a, B: BlockT, S: NetworkSpecialization, H: ExHashT> { - protocol: &'a mut Swarm, +struct NetworkLink<'a, B: BlockT, H: ExHashT> { + protocol: &'a mut Swarm, } -impl<'a, B: BlockT, S: NetworkSpecialization, H: ExHashT> Link for NetworkLink<'a, B, S, H> { +impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { fn blocks_processed( &mut self, imported: usize, diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index b11b1870511bed907f1b79def3836ad6579137c0..75ee2d5db89b9c64569c23a70c7384080819ef36 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -17,10 +17,10 @@ use futures::prelude::*; use libp2p::{ InboundUpgradeExt, OutboundUpgradeExt, PeerId, Transport, - mplex, identity, yamux, bandwidth, wasm_ext + mplex, identity, bandwidth, wasm_ext, noise }; #[cfg(not(target_os = "unknown"))] -use libp2p::{tcp, dns, websocket, noise}; +use libp2p::{tcp, dns, websocket}; use libp2p::core::{self, upgrade, transport::boxed::Boxed, transport::OptionalTransport, muxing::StreamMuxerBox}; use std::{io, sync::Arc, time::Duration, usize}; @@ -36,10 +36,10 @@ pub use self::bandwidth::BandwidthSinks; pub fn build_transport( keypair: identity::Keypair, memory_only: bool, - wasm_external_transport: Option + wasm_external_transport: Option, + use_yamux_flow_control: bool ) -> (Boxed<(PeerId, StreamMuxerBox), io::Error>, Arc) { // Build configuration objects for encryption mechanisms. - #[cfg(not(target_os = "unknown"))] let noise_config = { let noise_keypair = noise::Keypair::new().into_authentic(&keypair) // For more information about this panic, see in "On the Importance of Checking @@ -55,7 +55,15 @@ pub fn build_transport( let mut mplex_config = mplex::MplexConfig::new(); mplex_config.max_buffer_len_behaviour(mplex::MaxBufferBehaviour::Block); mplex_config.max_buffer_len(usize::MAX); - let yamux_config = yamux::Config::default(); + + let mut yamux_config = libp2p::yamux::Config::default(); + yamux_config.set_lazy_open(true); // Only set SYN flag on first data frame sent to the remote. + + if use_yamux_flow_control { + // Enable proper flow-control: window updates are only sent when + // buffered data has been consumed. + yamux_config.set_window_update_mode(libp2p::yamux::WindowUpdateMode::OnRead); + } // Build the base layer of the transport. let transport = if let Some(t) = wasm_external_transport { @@ -86,9 +94,6 @@ pub fn build_transport( let (transport, sinks) = bandwidth::BandwidthLogging::new(transport, Duration::from_secs(5)); // Encryption - - // For non-WASM, we support both secio and noise. - #[cfg(not(target_os = "unknown"))] let transport = transport.and_then(move |stream, endpoint| { core::upgrade::apply(stream, noise_config, endpoint, upgrade::Version::V1) .and_then(|(remote_id, out)| async move { @@ -100,15 +105,6 @@ pub fn build_transport( }) }); - // We refuse all WASM connections for now. It is intended that we negotiate noise in the - // future. See https://github.com/libp2p/rust-libp2p/issues/1414 - #[cfg(target_os = "unknown")] - let transport = transport.and_then(move |_, _| async move { - let r: Result<(wasm_ext::Connection, PeerId), _> = - Err(io::Error::new(io::ErrorKind::Other, format!("No encryption protocol supported"))); - r - }); - // Multiplexing let transport = transport.and_then(move |(stream, peer_id), endpoint| { let peer_id2 = peer_id.clone(); @@ -118,11 +114,12 @@ pub fn build_transport( core::upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1) .map_ok(|(id, muxer)| (id, core::muxing::StreamMuxerBox::new(muxer))) - }) + }); - .timeout(Duration::from_secs(20)) - .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) - .boxed(); + let transport = transport + .timeout(Duration::from_secs(20)) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) + .boxed(); (transport, sinks) } diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 3f1d1d5867707adbc0def59f597df307b44a82f0..cd9258cc08bc69508fc861b24de0b977b1e818c5 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -1,30 +1,33 @@ [package] description = "Integration tests for Substrate network protocol" name = "sc-network-test" -version = "0.8.0" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-network = { version = "0.8", path = "../" } +sc-network = { version = "0.8.0-alpha.1", path = "../" } log = "0.4.8" parking_lot = "0.10.0" futures = "0.1.29" futures03 = { package = "futures", version = "0.3.1", features = ["compat"] } -futures-timer = "0.4.0" +futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8", path = "../../" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sc-block-builder = { version = "0.8", path = "../../block-builder" } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } +libp2p = { version = "0.16.1", default-features = false, features = ["libp2p-websocket"] } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sc-client = { version = "0.8.0-alpha.1", path = "../../" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../../api" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sc-block-builder = { version = "0.8.0-alpha.1", path = "../../block-builder" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/babe" } env_logger = "0.7.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } tempfile = "3.1.0" tokio = "0.1.22" diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 22e340a2a81b39f220c2f4cf49c44c86f7f0f63e..d09897e853ec2074041bb31565af8efe21f5abfc 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -25,7 +25,7 @@ use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData}; use libp2p::build_multiaddr; use log::trace; -use sc_network::FinalityProofProvider; +use sc_network::config::FinalityProofProvider; use sp_blockchain::{ Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, Info as BlockchainInfo, }; @@ -52,17 +52,14 @@ use sc_network::config::{NetworkConfiguration, TransportConfig, BoxFinalityProof use libp2p::PeerId; use parking_lot::Mutex; use sp_core::H256; -use sc_network::ProtocolConfig; +use sc_network::config::{ProtocolConfig, TransactionPool}; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::Justification; -use sc_network::TransactionPool; -use sc_network::specialization::NetworkSpecialization; use substrate_test_runtime_client::{self, AccountKeyring}; pub use substrate_test_runtime_client::runtime::{Block, Extrinsic, Hash, Transfer}; pub use substrate_test_runtime_client::{TestClient, TestClientBuilder, TestClientBuilderExt}; -pub use sc_network::specialization::DummySpecialization; type AuthorityId = sp_consensus_babe::AuthorityId; @@ -85,21 +82,13 @@ impl Verifier for PassThroughVerifier { .or_else(|| l.try_as_raw(OpaqueDigestItemId::Consensus(b"babe"))) ) .map(|blob| vec![(well_known_cache_keys::AUTHORITIES, blob.to_vec())]); + let mut import = BlockImportParams::new(origin, header); + import.body = body; + import.finalized = self.0; + import.justification = justification; + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); - Ok((BlockImportParams { - origin, - header, - body, - storage_changes: None, - finalized: self.0, - justification, - post_digests: vec![], - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }, maybe_keys)) + Ok((import, maybe_keys)) } } @@ -186,23 +175,23 @@ impl PeersClient { } } -pub struct Peer> { +pub struct Peer { pub data: D, client: PeersClient, /// We keep a copy of the verifier so that we can invoke it for locally-generated blocks, /// instead of going through the import queue. - verifier: VerifierAdapter>, + verifier: VerifierAdapter, /// We keep a copy of the block_import so that we can invoke it for locally-generated blocks, /// instead of going through the import queue. block_import: BlockImportAdapter<()>, select_chain: Option>, backend: Option>, - network: NetworkWorker::Hash>, + network: NetworkWorker::Hash>, imported_blocks_stream: Box, Error = ()> + Send>, finality_notification_stream: Box, Error = ()> + Send>, } -impl> Peer { +impl Peer { /// Get this peer ID. pub fn id(&self) -> PeerId { self.network.service().local_peer_id() @@ -248,7 +237,7 @@ impl> Peer { where F: FnMut(BlockBuilder) -> Block { let best_hash = self.client.info().best_hash; - self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block) + self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block, false) } /// Add blocks to the peer -- edit the block before adding. The chain will @@ -258,7 +247,8 @@ impl> Peer { at: BlockId, count: usize, origin: BlockOrigin, - mut edit_block: F + mut edit_block: F, + headers_only: bool, ) -> H256 where F: FnMut(BlockBuilder) -> Block { let full_client = self.client.as_full() .expect("blocks could only be generated by full clients"); @@ -283,7 +273,7 @@ impl> Peer { origin, header.clone(), None, - Some(block.extrinsics) + if headers_only { None } else { Some(block.extrinsics) }, ).unwrap(); let cache = if let Some(cache) = cache { cache.into_iter().collect() @@ -291,7 +281,7 @@ impl> Peer { Default::default() }; self.block_import.import_block(import_block, cache).expect("block_import failed"); - self.network.on_block_imported(hash, header, Vec::new(), true); + self.network.on_block_imported(header, Vec::new(), true); at = hash; } @@ -305,28 +295,46 @@ impl> Peer { self.push_blocks_at(BlockId::Hash(best_hash), count, with_tx) } + /// Push blocks to the peer (simplified: with or without a TX) + pub fn push_headers(&mut self, count: usize) -> H256 { + let best_hash = self.client.info().best_hash; + self.generate_tx_blocks_at(BlockId::Hash(best_hash), count, false, true) + } + /// Push blocks to the peer (simplified: with or without a TX) starting from /// given hash. pub fn push_blocks_at(&mut self, at: BlockId, count: usize, with_tx: bool) -> H256 { + self.generate_tx_blocks_at(at, count, with_tx, false) + } + + /// Push blocks/headers to the peer (simplified: with or without a TX) starting from + /// given hash. + fn generate_tx_blocks_at(&mut self, at: BlockId, count: usize, with_tx: bool, headers_only:bool) -> H256 { let mut nonce = 0; if with_tx { - self.generate_blocks_at(at, count, BlockOrigin::File, |mut builder| { - let transfer = Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Alice.into(), - amount: 1, - nonce, - }; - builder.push(transfer.into_signed_tx()).unwrap(); - nonce = nonce + 1; - builder.build().unwrap().block - }) + self.generate_blocks_at( + at, + count, + BlockOrigin::File, |mut builder| { + let transfer = Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Alice.into(), + amount: 1, + nonce, + }; + builder.push(transfer.into_signed_tx()).unwrap(); + nonce = nonce + 1; + builder.build().unwrap().block + }, + headers_only + ) } else { self.generate_blocks_at( at, count, BlockOrigin::File, |builder| builder.build().unwrap().block, + headers_only, ) } } @@ -344,7 +352,7 @@ impl> Peer { } /// Get a reference to the network service. - pub fn network_service(&self) -> &Arc::Hash>> { + pub fn network_service(&self) -> &Arc::Hash>> { &self.network.service() } @@ -368,6 +376,11 @@ impl> Peer { |backend| backend.blocks_count() ).unwrap_or(0) } + + /// Return a collection of block hashes that failed verification + pub fn failed_verifications(&self) -> HashMap<::Hash, String> { + self.verifier.failed_verifications.lock().clone() + } } pub struct EmptyTransactionPool; @@ -395,16 +408,6 @@ impl TransactionPool for EmptyTransactionPool { fn transaction(&self, _h: &Hash) -> Option { None } } -pub trait SpecializationFactory { - fn create() -> Self; -} - -impl SpecializationFactory for DummySpecialization { - fn create() -> DummySpecialization { - DummySpecialization - } -} - /// Implements `BlockImport` for any `Transaction`. Internally the transaction is /// "converted", aka the field is set to `None`. /// @@ -493,15 +496,13 @@ impl BlockImport for BlockImportAdapter { } /// Implements `Verifier` on an `Arc>`. Used internally. -struct VerifierAdapter(Arc>>); - -impl Clone for VerifierAdapter { - fn clone(&self) -> Self { - VerifierAdapter(self.0.clone()) - } +#[derive(Clone)] +struct VerifierAdapter { + verifier: Arc>>>, + failed_verifications: Arc>>, } -impl> Verifier for VerifierAdapter { +impl Verifier for VerifierAdapter { fn verify( &mut self, origin: BlockOrigin, @@ -509,12 +510,24 @@ impl> Verifier for VerifierAdapter { justification: Option, body: Option> ) -> Result<(BlockImportParams, Option)>>), String> { - self.0.lock().verify(origin, header, justification, body) + let hash = header.hash(); + self.verifier.lock().verify(origin, header, justification, body).map_err(|e| { + self.failed_verifications.lock().insert(hash, e.clone()); + e + }) + } +} + +impl VerifierAdapter { + fn new(verifier: Arc>>>) -> VerifierAdapter { + VerifierAdapter { + verifier, + failed_verifications: Default::default(), + } } } pub trait TestNetFactory: Sized { - type Specialization: NetworkSpecialization + SpecializationFactory; type Verifier: 'static + Verifier; type PeerData: Default; @@ -528,9 +541,9 @@ pub trait TestNetFactory: Sized { ) -> Self::Verifier; /// Get reference to peer. - fn peer(&mut self, i: usize) -> &mut Peer; - fn peers(&self) -> &Vec>; - fn mut_peers>)>( + fn peer(&mut self, i: usize) -> &mut Peer; + fn peers(&self) -> &Vec>; + fn mut_peers>)>( &mut self, closure: F, ); @@ -600,7 +613,7 @@ pub trait TestNetFactory: Sized { config, &data, ); - let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); + let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); let import_queue = Box::new(BasicQueue::new( verifier.clone(), @@ -628,7 +641,6 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - specialization: self::SpecializationFactory::create(), block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())) }).unwrap(); @@ -676,7 +688,7 @@ pub trait TestNetFactory: Sized { &config, &data, ); - let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); + let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>))); let import_queue = Box::new(BasicQueue::new( verifier.clone(), @@ -704,7 +716,6 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - specialization: self::SpecializationFactory::create(), block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())) }).unwrap(); @@ -756,13 +767,37 @@ pub trait TestNetFactory: Sized { Async::Ready(()) } + /// Polls the testnet until theres' no activiy of any kind. + /// + /// Must be executed in a task context. + fn poll_until_idle(&mut self) -> Async<()> { + self.poll(); + + for peer in self.peers().iter() { + if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 { + return Async::NotReady + } + if peer.network.num_sync_requests() != 0 { + return Async::NotReady + } + } + Async::Ready(()) + } + /// Blocks the current thread until we are sync'ed. /// - /// Calls `poll_until_sync` repeatidely with the runtime passed as parameter. + /// Calls `poll_until_sync` repeatedly with the runtime passed as parameter. fn block_until_sync(&mut self, runtime: &mut tokio::runtime::current_thread::Runtime) { runtime.block_on(futures::future::poll_fn::<(), (), _>(|| Ok(self.poll_until_sync()))).unwrap(); } + /// Blocks the current thread until there are no pending packets. + /// + /// Calls `poll_until_idle` repeatedly with the runtime passed as parameter. + fn block_until_idle(&mut self, runtime: &mut tokio::runtime::current_thread::Runtime) { + runtime.block_on(futures::future::poll_fn::<(), (), _>(|| Ok(self.poll_until_idle()))).unwrap(); + } + /// Polls the testnet. Processes all the pending actions and returns `NotReady`. fn poll(&mut self) { self.mut_peers(|peers| { @@ -776,7 +811,6 @@ pub trait TestNetFactory: Sized { // We poll `imported_blocks_stream`. while let Ok(Async::Ready(Some(notification))) = peer.imported_blocks_stream.poll() { peer.network.on_block_imported( - notification.hash, notification.header, Vec::new(), true, @@ -797,11 +831,10 @@ pub trait TestNetFactory: Sized { } pub struct TestNet { - peers: Vec>, + peers: Vec>, } impl TestNetFactory for TestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = (); @@ -818,15 +851,15 @@ impl TestNetFactory for TestNet { PassThroughVerifier(false) } - fn peer(&mut self, i: usize) -> &mut Peer<(), Self::Specialization> { + fn peer(&mut self, i: usize) -> &mut Peer<()> { &mut self.peers[i] } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { &self.peers } - fn mut_peers>)>(&mut self, closure: F) { + fn mut_peers>)>(&mut self, closure: F) { closure(&mut self.peers); } } @@ -850,7 +883,6 @@ impl JustificationImport for ForceFinalized { pub struct JustificationTestNet(TestNet); impl TestNetFactory for JustificationTestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = (); @@ -862,17 +894,16 @@ impl TestNetFactory for JustificationTestNet { self.0.make_verifier(client, config, peer_data) } - fn peer(&mut self, i: usize) -> &mut Peer { + fn peer(&mut self, i: usize) -> &mut Peer { self.0.peer(i) } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { self.0.peers() } fn mut_peers>, + &mut Vec>, )>(&mut self, closure: F) { self.0.mut_peers(closure) } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 2140de0973449a677abe4547a4830f1f8378626a..2094ddae60c606500aec757ad79ea47c8f9ea4fe 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -405,7 +405,7 @@ fn blocks_are_not_announced_by_light_nodes() { net.peers.remove(0); // Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together. - let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).compat(); + let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { net.poll(); delay.poll().map_err(|_| ()) @@ -504,7 +504,7 @@ fn can_not_sync_from_light_peer() { net.peers.remove(0); // ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds - let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).compat(); + let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { net.poll(); test_finished.poll().map_err(|_| ()) @@ -660,3 +660,24 @@ fn does_not_sync_announced_old_best_block() { })).unwrap(); assert!(!net.peer(1).is_major_syncing()); } + +#[test] +fn full_sync_requires_block_body() { + // Check that we don't sync headers-only in full mode. + let _ = ::env_logger::try_init(); + let mut runtime = current_thread::Runtime::new().unwrap(); + let mut net = TestNet::new(2); + + net.peer(0).push_headers(1); + // Wait for nodes to connect + runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { + net.poll(); + if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { + Ok(Async::NotReady) + } else { + Ok(Async::Ready(())) + } + })).unwrap(); + net.block_until_idle(&mut runtime); + assert_eq!(net.peer(1).client.info().best_number, 0); +} diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index bdbf3fa1995e411ad9d2cb79879cc256b99c749b..ba7cee68112ea4bc32ab3e324a5e630551f56c5f 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -1,42 +1,43 @@ [package] description = "Substrate offchain workers" name = "sc-offchain" -version = "2.0.0" +version = "2.0.0-alpha.1" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -bytes = "0.4.12" -sc-client-api = { version = "2.0.0", path = "../api" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } +bytes = "0.5" +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } fnv = "1.0.6" -futures01 = { package = "futures", version = "0.1" } futures = "0.3.1" -futures-timer = "2.0" +futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" -sp-offchain = { version = "2.0.0", path = "../../primitives/offchain" } +sp-offchain = { version = "2.0.0-alpha.1", path = "../../primitives/offchain" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } rand = "0.7.2" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-network = { version = "0.8", path = "../network" } -sc-keystore = { version = "2.0.0", path = "../keystore" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../keystore" } [target.'cfg(not(target_os = "unknown"))'.dependencies] -hyper = "0.12.35" -hyper-rustls = "0.17.1" +hyper = "0.13.2" +hyper-rustls = "0.19" [dev-dependencies] -sc-client-db = { version = "0.8", default-features = true, path = "../db/" } +sc-client-db = { version = "0.8.0-alpha.1", default-features = true, path = "../db/" } env_logger = "0.7.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -tokio = "0.1.22" -sc-transaction-pool = { version = "2.0.0", path = "../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } +tokio = "0.2" +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } [features] default = [] diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index 84c3ecd69b37ff7956bbe16e1b23f8383ad356b5..0483f84e2f114f4af17e38533964483ecda5e056 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -18,7 +18,7 @@ //! function returns a pair of [`HttpApi`] and [`HttpWorker`] that share some state. //! //! The [`HttpApi`] is (indirectly) passed to the runtime when calling an offchain worker, while -//! the [`HttpWorker`] must be processed in the background. The [`HttpApi`] mimicks the API of the +//! the [`HttpWorker`] must be processed in the background. The [`HttpApi`] mimics the API of the //! HTTP-related methods available to offchain workers. //! //! The reason for this design is driven by the fact that HTTP requests should continue running @@ -26,9 +26,9 @@ //! actively calling any function. use crate::api::timestamp; -use bytes::Buf as _; +use bytes::buf::ext::{Reader, BufExt}; use fnv::FnvHashMap; -use futures::{prelude::*, channel::mpsc, compat::Compat01As03}; +use futures::{prelude::*, future, channel::mpsc}; use log::error; use sp_core::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError}; use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll}; @@ -50,7 +50,7 @@ pub fn http() -> (HttpApi, HttpWorker) { let engine = HttpWorker { to_api, from_api, - http_client: hyper::Client::builder().build(hyper_rustls::HttpsConnector::new(1)), + http_client: hyper::Client::builder().build(hyper_rustls::HttpsConnector::new()), requests: Vec::new(), }; @@ -103,14 +103,14 @@ struct HttpApiRequestRp { /// Elements extracted from the channel are first put into `current_read_chunk`. /// If the channel produces an error, then that is translated into an `IoError` and the request /// is removed from the list. - body: stream::Fuse>>, + body: stream::Fuse>>, /// Chunk that has been extracted from the channel and that is currently being read. /// Reading data from the response should read from this field in priority. - current_read_chunk: Option>, + current_read_chunk: Option>, } impl HttpApi { - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_start( &mut self, method: &str, @@ -122,7 +122,7 @@ impl HttpApi { let (body_sender, body) = hyper::Body::channel(); let mut request = hyper::Request::new(body); *request.method_mut() = hyper::Method::from_bytes(method.as_bytes()).map_err(|_| ())?; - *request.uri_mut() = hyper::Uri::from_shared(From::from(uri)).map_err(|_| ())?; + *request.uri_mut() = hyper::Uri::from_maybe_shared(uri.to_owned()).map_err(|_| ())?; let new_id = self.next_id; debug_assert!(!self.requests.contains_key(&new_id)); @@ -138,7 +138,7 @@ impl HttpApi { Ok(new_id) } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_add_header( &mut self, request_id: HttpRequestId, @@ -158,7 +158,7 @@ impl HttpApi { Ok(()) } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_write_body( &mut self, request_id: HttpRequestId, @@ -177,9 +177,7 @@ impl HttpApi { // (if the body has been written), or `DeadlineReached`, or `IoError`. // If `IoError` is returned, don't forget to remove the request from the list. let mut poll_sender = move |sender: &mut hyper::body::Sender| -> Result<(), HttpError> { - let mut when_ready = future::maybe_done(Compat01As03::new( - futures01::future::poll_fn(|| sender.poll_ready()) - )); + let mut when_ready = future::maybe_done(future::poll_fn(|cx| sender.poll_ready(cx))); futures::executor::block_on(future::select(&mut when_ready, &mut deadline)); match when_ready { future::MaybeDone::Done(Ok(())) => {} @@ -191,13 +189,11 @@ impl HttpApi { } }; - match sender.send_data(hyper::Chunk::from(chunk.to_owned())) { - Ok(()) => Ok(()), - Err(_chunk) => { + futures::executor::block_on(sender.send_data(hyper::body::Bytes::from(chunk.to_owned()))) + .map_err(|_| { error!("HTTP sender refused data despite being ready"); - Err(HttpError::IoError) - }, - } + HttpError::IoError + }) }; loop { @@ -266,7 +262,7 @@ impl HttpApi { } } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_wait( &mut self, ids: &[HttpRequestId], @@ -392,7 +388,7 @@ impl HttpApi { } } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_headers( &mut self, request_id: HttpRequestId @@ -411,7 +407,7 @@ impl HttpApi { .collect() } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_read_body( &mut self, request_id: HttpRequestId, @@ -538,7 +534,7 @@ enum WorkerToApi { /// the next item. /// Can also be used to send an error, in case an error happend on the HTTP socket. After /// an error is sent, the channel will close. - body: mpsc::Receiver>, + body: mpsc::Receiver>, }, /// A request has failed because of an error. The request is then no longer valid. Fail { @@ -564,13 +560,13 @@ pub struct HttpWorker { /// HTTP request being processed by the worker. enum HttpWorkerRequest { /// Request has been dispatched and is waiting for a response from the Internet. - Dispatched(Compat01As03), + Dispatched(hyper::client::ResponseFuture), /// Progressively reading the body of the response and sending it to the channel. ReadBody { /// Body to read `Chunk`s from. Only used if the channel is ready to accept data. - body: Compat01As03, + body: hyper::Body, /// Channel to the [`HttpApi`] where we send the chunks to. - tx: mpsc::Sender>, + tx: mpsc::Sender>, }, } @@ -608,7 +604,7 @@ impl Future for HttpWorker { // We received a response! Decompose it into its parts. let status_code = response.status(); let headers = mem::replace(response.headers_mut(), hyper::HeaderMap::new()); - let body = Compat01As03::new(response.into_body()); + let body = response.into_body(); let (body_tx, body_rx) = mpsc::channel(3); let _ = me.to_api.unbounded_send(WorkerToApi::Response { @@ -660,7 +656,7 @@ impl Future for HttpWorker { Poll::Pending => {}, Poll::Ready(None) => return Poll::Ready(()), // stops the worker Poll::Ready(Some(ApiToWorker::Dispatch { id, request })) => { - let future = Compat01As03::new(me.http_client.request(request)); + let future = me.http_client.request(request); debug_assert!(me.requests.iter().all(|(i, _)| *i != id)); me.requests.push((id, HttpWorkerRequest::Dispatched(future))); cx.waker().wake_by_ref(); // reschedule the task to poll the request @@ -692,31 +688,36 @@ impl fmt::Debug for HttpWorkerRequest { #[cfg(test)] mod tests { + use core::convert::Infallible; use crate::api::timestamp; use super::http; - use futures::prelude::*; - use futures01::Future as _; use sp_core::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Duration}; // Returns an `HttpApi` whose worker is ran in the background, and a `SocketAddr` to an HTTP // server that runs in the background as well. macro_rules! build_api_server { () => {{ + fn tokio_run(future: impl std::future::Future) { + let _ = tokio::runtime::Runtime::new().unwrap().block_on(future); + } + let (api, worker) = http(); - // Note: we have to use tokio because hyper still uses old futures. - std::thread::spawn(move || { - tokio::run(futures::compat::Compat::new(worker.map(|()| Ok::<(), ()>(())))) - }); + std::thread::spawn(move || tokio_run(worker)); + let (addr_tx, addr_rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { - let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()) - .serve(|| { - hyper::service::service_fn_ok(move |_: hyper::Request| { - hyper::Response::new(hyper::Body::from("Hello World!")) - }) - }); - let _ = addr_tx.send(server.local_addr()); - hyper::rt::run(server.map_err(|e| panic!("{:?}", e))); + tokio_run(async move { + let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()) + .serve(hyper::service::make_service_fn(|_| { async move { + Ok::<_, Infallible>(hyper::service::service_fn(move |_req| async move { + Ok::<_, Infallible>( + hyper::Response::new(hyper::Body::from("Hello World!")) + ) + })) + }})); + let _ = addr_tx.send(server.local_addr()); + server.await + }); }); (api, addr_rx.recv().unwrap()) }}; @@ -947,7 +948,7 @@ mod tests { #[test] fn fuzzing() { - // Uses the API in random ways to try to trigger panicks. + // Uses the API in random ways to try to trigger panics. // Doesn't test some paths, such as waiting for multiple requests. Also doesn't test what // happens if the server force-closes our socket. diff --git a/client/offchain/src/api/http_dummy.rs b/client/offchain/src/api/http_dummy.rs index 8725c8989979582d455b65d3e93d07bcf1eac5e0..5ff77a1068312bbc5da2f71d9c15a9dee7bd8380 100644 --- a/client/offchain/src/api/http_dummy.rs +++ b/client/offchain/src/api/http_dummy.rs @@ -33,7 +33,7 @@ pub struct HttpApi; pub struct HttpWorker; impl HttpApi { - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_start( &mut self, _: &str, @@ -43,7 +43,7 @@ impl HttpApi { Err(()) } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_add_header( &mut self, _: HttpRequestId, @@ -54,7 +54,7 @@ impl HttpApi { never be called; qed") } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn request_write_body( &mut self, _: HttpRequestId, @@ -65,7 +65,7 @@ impl HttpApi { never be called; qed") } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_wait( &mut self, requests: &[HttpRequestId], @@ -79,7 +79,7 @@ impl HttpApi { } } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_headers( &mut self, _: HttpRequestId @@ -88,7 +88,7 @@ impl HttpApi { never be called; qed") } - /// Mimicks the corresponding method in the offchain API. + /// Mimics the corresponding method in the offchain API. pub fn response_read_body( &mut self, _: HttpRequestId, diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index f0a10a540990d4937ca224faa8892535bf808643..7fc9671fcbcd826c85b69c9414fbaa34e56797da 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -97,7 +97,7 @@ impl OffchainWorkers< is_validator: bool, ) -> impl Future { let runtime = self.client.runtime_api(); - let at = BlockId::number(*header.number()); + let at = BlockId::hash(header.hash()); let has_api_v1 = runtime.has_api_with::, _>( &at, |v| v == 1 ); @@ -169,7 +169,6 @@ mod tests { use substrate_test_runtime_client::runtime::Block; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; - use sp_runtime::{generic::Header, traits::Header as _}; struct MockNetworkStateInfo(); @@ -205,18 +204,12 @@ mod tests { let pool = Arc::new(TestPool(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), - ))); + ).0)); client.execution_extensions() .register_transaction_pool(Arc::downgrade(&pool.clone()) as _); let db = sc_client_db::offchain::LocalStorage::new_test(); let network_state = Arc::new(MockNetworkStateInfo()); - let header = Header::new( - 0u64, - Default::default(), - Default::default(), - Default::default(), - Default::default(), - ); + let header = client.header(&BlockId::number(0)).unwrap().unwrap(); // when let offchain = OffchainWorkers::new(client, db); @@ -224,6 +217,6 @@ mod tests { // then assert_eq!(pool.0.status().ready, 1); - assert_eq!(pool.0.ready().next().unwrap().is_propagateable(), false); + assert_eq!(pool.0.ready().next().unwrap().is_propagable(), false); } } diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 85c988d048957b1b42977359fbc2b600aebabe8b..2ba10466fb59be4c8605da21e6e36fbc34967e0a 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -3,15 +3,17 @@ description = "Connectivity manager based on reputation" homepage = "http://parity.io" license = "GPL-3.0" name = "sc-peerset" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" +repository = "https://github.com/paritytech/substrate/" [dependencies] futures = "0.3.1" -libp2p = { version = "0.15.0", default-features = false } +libp2p = { version = "0.16.1", default-features = false } log = "0.4.8" serde_json = "1.0.41" +wasm-timer = "0.2" [dev-dependencies] rand = "0.7.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index c9589127270071ebfede9a67faf68a982e8c2823..53e3b35297b267e09a1ccf8c4f02f2ca77efe0f9 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -19,17 +19,19 @@ mod peersstate; -use std::{collections::{HashSet, HashMap}, collections::VecDeque, time::Instant}; +use std::{collections::{HashSet, HashMap}, collections::VecDeque}; use futures::{prelude::*, channel::mpsc}; -use libp2p::PeerId; use log::{debug, error, trace}; use serde_json::json; use std::{pin::Pin, task::Context, task::Poll}; +use wasm_timer::Instant; + +pub use libp2p::PeerId; /// We don't accept nodes whose reputation is under this value. const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100); /// Reputation change for a node when we get disconnected from it. -const DISCONNECT_REPUTATION_CHANGE: i32 = -10; +const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// Reserved peers group ID const RESERVED_NODES: &'static str = "reserved"; @@ -44,7 +46,7 @@ enum Action { RemoveFromPriorityGroup(String, PeerId), } -/// Shared handle to the peer set manager (PSM). Distributed around the code. +/// Description of a reputation adjustment for a node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReputationChange { /// Reputation delta. @@ -198,14 +200,16 @@ impl Peerset { tx: tx.clone(), }; + let now = Instant::now(); + let mut peerset = Peerset { data: peersstate::PeersState::new(config.in_peers, config.out_peers, config.reserved_only), tx, rx, reserved_only: config.reserved_only, message_queue: VecDeque::new(), - created: Instant::now(), - latest_time_update: Instant::now(), + created: now, + latest_time_update: now, }; peerset.data.set_priority_group(RESERVED_NODES, config.reserved_nodes.into_iter().collect()); diff --git a/client/peerset/src/peersstate.rs b/client/peerset/src/peersstate.rs index a27d6e616b23e1eb70026572c3c2fd1045cec30d..96a6698734b744436cc846fcc772e2edb53b221a 100644 --- a/client/peerset/src/peersstate.rs +++ b/client/peerset/src/peersstate.rs @@ -156,7 +156,7 @@ impl PeersState { pub fn priority_not_connected_peer(&mut self) -> Option { let id = self.priority_nodes.values() .flatten() - .find(|id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) + .find(|&id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) .cloned(); id.map(move |id| NotConnectedPeer { state: self, @@ -170,7 +170,7 @@ impl PeersState { pub fn priority_not_connected_peer_from_group(&mut self, group_id: &str) -> Option { let id = self.priority_nodes.get(group_id) .and_then(|group| group.iter() - .find(|id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) + .find(|&id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) .cloned()); id.map(move |id| NotConnectedPeer { state: self, @@ -300,7 +300,7 @@ impl PeersState { for id in &peers { // update slots for nodes that become priority - if !all_other_groups.contains(&id) { + if !all_other_groups.contains(id) { let peer = self.nodes.entry(id.clone()).or_default(); match peer.connection_state { ConnectionState::In => self.num_in -= 1, @@ -322,7 +322,7 @@ impl PeersState { /// Remove a peer from a priority group. pub fn remove_from_priority_group(&mut self, group_id: &str, peer_id: &PeerId) { let mut peers = self.priority_nodes.get(group_id).cloned().unwrap_or_default(); - peers.remove(&peer_id); + peers.remove(peer_id); self.set_priority_group(group_id, peers); } diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index b591d83d8f2af9ec0caaa7a4fc1038a7e8f37f82..c2b0b44a3a95d43f30210efc73563e4e0ae520a3 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -108,7 +108,7 @@ fn test_once() { // If we generate 4, connect to a random node. 4 => if let Some(id) = known_nodes.iter() - .filter(|n| incoming_nodes.values().all(|m| m != *n) && !connected_nodes.contains(n)) + .filter(|n| incoming_nodes.values().all(|m| m != *n) && !connected_nodes.contains(*n)) .choose(&mut rng) { peerset.incoming(id.clone(), next_incoming_id.clone()); incoming_nodes.insert(next_incoming_id.clone(), id.clone()); @@ -120,7 +120,7 @@ fn test_once() { 6 => peerset_handle.set_reserved_only(false), // 7 and 8 are about switching a random node in or out of reserved mode. - 7 => if let Some(id) = known_nodes.iter().filter(|n| !reserved_nodes.contains(n)).choose(&mut rng) { + 7 => if let Some(id) = known_nodes.iter().filter(|n| !reserved_nodes.contains(*n)).choose(&mut rng) { peerset_handle.add_reserved_peer(id.clone()); reserved_nodes.insert(id.clone()); } diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 4781c9d35096ca02e68f0e6d59b2ca963bed6e24..7ab70df55ccea04d646e062c3ff79c719a504b8c 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-rpc-api" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0" } @@ -15,9 +17,10 @@ jsonrpc-derive = "14.0.3" jsonrpc-pubsub = "14.0.3" log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-version = { version = "2.0.0-alpha.1", path = "../../primitives/version" } +sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-alpha.1"} serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sp-rpc = { version = "2.0.0", path = "../../primitives/rpc" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } +sp-rpc = { version = "2.0.0-alpha.1", path = "../../primitives/rpc" } diff --git a/client/rpc-api/src/author/mod.rs b/client/rpc-api/src/author/mod.rs index 4fbf0c73a366696109b4efc4043f7171479991c9..49c4c996fa9e1705fa4344bcce708ecf66d5b45d 100644 --- a/client/rpc-api/src/author/mod.rs +++ b/client/rpc-api/src/author/mod.rs @@ -77,7 +77,7 @@ pub trait AuthorApi { /// Submit an extrinsic to watch. /// /// See [`TransactionStatus`](sp_transaction_pool::TransactionStatus) for details on transaction - /// lifecycle. + /// life cycle. #[pubsub( subscription = "author_extrinsicUpdate", subscribe, diff --git a/client/rpc-api/src/errors.rs b/client/rpc-api/src/errors.rs index 9db41d0497068abfdfe67635352e2f8b06aac227..b75c34ead3806a7a0be3bb8104c4a30cd961138e 100644 --- a/client/rpc-api/src/errors.rs +++ b/client/rpc-api/src/errors.rs @@ -20,7 +20,7 @@ pub fn internal(e: E) -> jsonrpc_core::Error { warn!("Unknown error: {:?}", e); jsonrpc_core::Error { code: jsonrpc_core::ErrorCode::InternalError, - message: "Unknown error occured".into(), + message: "Unknown error occurred".into(), data: Some(format!("{:?}", e).into()), } } diff --git a/client/rpc-api/src/subscriptions.rs b/client/rpc-api/src/subscriptions.rs index 808c9d5ba449019052f467464718862f346a1d98..54881bad5123bfb9c98c0b716f7b1b514bab2d40 100644 --- a/client/rpc-api/src/subscriptions.rs +++ b/client/rpc-api/src/subscriptions.rs @@ -71,7 +71,7 @@ impl Subscriptions { /// Borrows the internal task executor. /// - /// This can be used to spawn additional tasks on the underyling event loop. + /// This can be used to spawn additional tasks on the underlying event loop. pub fn executor(&self) -> &TaskExecutor { &self.executor } diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index 6b0d3f4adacf8bc3d5d07b28082455916080df07..1bec566f52bcfb4d3b44932130989f4e156d4812 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-rpc-server" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] jsonrpc-core = "14.0.3" @@ -11,7 +13,7 @@ pubsub = { package = "jsonrpc-pubsub", version = "14.0.3" } log = "0.4.8" serde = "1.0.101" serde_json = "1.0.41" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } [target.'cfg(not(target_os = "unknown"))'.dependencies] http = { package = "jsonrpc-http-server", version = "14.0.3" } diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index d09617ebc1658405e87fa127827987d6ddee9d94..d4f618f0064aae3010550ac808ded6c3e022ac01 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -1,41 +1,43 @@ [package] name = "sc-rpc" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-rpc-api = { version = "0.8", path = "../rpc-api" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } +sc-rpc-api = { version = "0.8.0-alpha.1", path = "../rpc-api" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.0.0" } futures = { version = "0.3.1", features = ["compat"] } jsonrpc-pubsub = "14.0.3" log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } rpc = { package = "jsonrpc-core", version = "14.0.3" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } +sp-version = { version = "2.0.0-alpha.1", path = "../../primitives/version" } serde_json = "1.0.41" -sp-session = { version = "2.0.0", path = "../../primitives/session" } -sp-offchain = { version = "2.0.0", path = "../../primitives/offchain" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-rpc = { version = "2.0.0", path = "../../primitives/rpc" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8", path = "../executor" } -sc-keystore = { version = "2.0.0", path = "../keystore" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-session = { version = "2.0.0-alpha.1", path = "../../primitives/session" } +sp-offchain = { version = "2.0.0-alpha.1", path = "../../primitives/offchain" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-rpc = { version = "2.0.0-alpha.1", path = "../../primitives/rpc" } +sp-state-machine = { version = "0.8.0-alpha.1", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-alpha.1", path = "../executor" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../keystore" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } hash-db = { version = "0.15.2", default-features = false } parking_lot = "0.10.0" [dev-dependencies] assert_matches = "1.3.0" futures01 = { package = "futures", version = "0.1.29" } -sc-network = { version = "0.8", path = "../network" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } rustc-hex = "2.0.1" -sp-io = { version = "2.0.0", path = "../../primitives/io" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-io = { version = "2.0.0-alpha.1", path = "../../primitives/io" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.1.22" -sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" } +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../transaction-pool" } diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index ba9b9d344c2a2a80f3f287004470f7f52a493bcb..41bfc46d388bd5e64b3efc63f5645394916a1db1 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -64,7 +64,7 @@ impl Default for TestSetup { let pool = Arc::new(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), - )); + ).0); TestSetup { runtime: runtime::Runtime::new().expect("Failed to create runtime in test setup"), client, diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index a0ab11e9772045a78ec801892257947367ee8f2d..6508d46ddde65c2be0960364122d0e2810576a58 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -402,7 +402,7 @@ fn should_return_runtime_version() { let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ \"specVersion\":1,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",2],\ - [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",1],[\"0x40fe3ad401f8959a\",4],\ + [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",1],[\"0x40fe3ad401f8959a\",5],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",1],\ [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]]}"; diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index a280110093b0bbf99c0e0a76fdd0e2e0f35aad8b..4487566e44cfd9fc18b41112d9ec66555c728196 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -69,7 +69,7 @@ fn api>>(sync: T) -> System { let _ = sender.send(peers); } Request::NetworkState(sender) => { - let _ = sender.send(serde_json::to_value(&sc_network::NetworkState { + let _ = sender.send(serde_json::to_value(&sc_network::network_state::NetworkState { peer_id: String::new(), listened_addresses: Default::default(), external_addresses: Default::default(), @@ -223,8 +223,8 @@ fn system_peers() { fn system_network_state() { let res = wait_receiver(api(None).system_network_state()); assert_eq!( - serde_json::from_value::(res).unwrap(), - sc_network::NetworkState { + serde_json::from_value::(res).unwrap(), + sc_network::network_state::NetworkState { peer_id: String::new(), listened_addresses: Default::default(), external_addresses: Default::default(), diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index bc246db03e85caee47ed929c3789a450e1b8d8cf..53c7739b09c7ffc9390317283c4a6ca5a01198e2 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-service" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [features] default = ["rocksdb"] @@ -23,44 +25,44 @@ parking_lot = "0.10.0" lazy_static = "1.4.0" log = "0.4.8" slog = { version = "2.5.2", features = ["nested-values"] } -tokio-executor = "0.1.8" -futures-timer = "2" +futures-timer = "3.0.1" +wasm-timer = "0.2" exit-future = "0.2.0" serde = "1.0.101" serde_json = "1.0.41" sysinfo = "0.9.5" target_info = "0.1.0" -sc-keystore = { version = "2.0.0", path = "../keystore" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-session = { version = "2.0.0", path = "../../primitives/session" } -sp-application-crypto = { version = "2.0.0", path = "../../primitives/application-crypto" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sc-network = { version = "0.8", path = "../network" } -sc-chain-spec = { version = "2.0.0", path = "../chain-spec" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sc-client-db = { version = "0.8", path = "../db" } +sc-keystore = { version = "2.0.0-alpha.1", path = "../keystore" } +sp-io = { version = "2.0.0-alpha.1", path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-session = { version = "2.0.0-alpha.1", path = "../../primitives/session" } +sp-application-crypto = { version = "2.0.0-alpha.1", path = "../../primitives/application-crypto" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sc-network = { version = "0.8.0-alpha.1", path = "../network" } +sc-chain-spec = { version = "2.0.0-alpha.1", path = "../chain-spec" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sc-client = { version = "0.8.0-alpha.1", path = "../" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } +sc-client-db = { version = "0.8.0-alpha.1", path = "../db" } codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor = { version = "0.8", path = "../executor" } -sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-rpc-server = { version = "2.0.0", path = "../rpc-servers" } -sc-rpc = { version = "2.0.0", path = "../rpc" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sc-offchain = { version = "2.0.0", path = "../offchain" } -parity-multiaddr = { package = "parity-multiaddr", version = "0.7.1" } -grafana-data-source = { version = "0.8", path = "../../utils/grafana-data-source" } -sc-tracing = { version = "2.0.0", path = "../tracing" } +sc-executor = { version = "0.8.0-alpha.1", path = "../executor" } +sc-transaction-pool = { version = "2.0.0-alpha.1", path = "../transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } +sc-rpc-server = { version = "2.0.0-alpha.1", path = "../rpc-servers" } +sc-rpc = { version = "2.0.0-alpha.1", path = "../rpc" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } +sc-offchain = { version = "2.0.0-alpha.1", path = "../offchain" } +parity-multiaddr = { package = "parity-multiaddr", version = "0.5.0" } +prometheus-exporter = { path = "../../utils/prometheus" , version = "0.8.0-alpha.1"} +sc-tracing = { version = "2.0.0-alpha.1", path = "../tracing" } tracing = "0.1.10" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../finality-grandpa" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/babe" } +grandpa = { version = "0.8.0-alpha.1", package = "sc-finality-grandpa", path = "../finality-grandpa" } +grandpa-primitives = { version = "2.0.0-alpha.1", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } tokio = { version = "0.2", features = ["rt-core"] } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 2c10804675d3b13a6f898adcdb6c659e6993e453..7159e532c42598c68910f2d57665b9055c6e07dc 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID}; +use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; use crate::{SpawnTaskHandle, start_rpc_servers, build_network_future, TransactionPoolAdapter}; use crate::status_sinks; use crate::config::{Configuration, DatabaseConfig, KeystoreConfig}; @@ -34,25 +34,64 @@ use futures::{ }; use sc_keystore::{Store as Keystore}; use log::{info, warn, error}; -use sc_network::{FinalityProofProvider, OnDemand, NetworkService, NetworkStateInfo}; -use sc_network::{config::BoxFinalityProofRequestBuilder, specialization::NetworkSpecialization}; +use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; +use sc_network::{NetworkService, NetworkStateInfo}; use parking_lot::{Mutex, RwLock}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, NumberFor, SaturatedConversion, HasherFor, + Block as BlockT, NumberFor, SaturatedConversion, HasherFor, UniqueSaturatedInto, }; use sp_api::ProvideRuntimeApi; use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use std::{ borrow::Cow, io::{Read, Write, Seek}, - marker::PhantomData, sync::Arc, time::SystemTime, pin::Pin + marker::PhantomData, sync::Arc, pin::Pin }; +use wasm_timer::SystemTime; use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; -use sp_transaction_pool::MaintainedTransactionPool; +use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sp_blockchain; -use grafana_data_source::{self, record_metrics}; +use prometheus_exporter::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; + +struct ServiceMetrics { + block_height_number: GaugeVec, + peers_count: Gauge, + ready_transactions_number: Gauge, + memory_usage_bytes: Gauge, + cpu_usage_percentage: Gauge, + network_per_sec_bytes: GaugeVec, +} + +impl ServiceMetrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + block_height_number: register(GaugeVec::new( + Opts::new("block_height_number", "Height of the chain"), + &["status"] + )?, registry)?, + peers_count: register(Gauge::new( + "peers_count", "Number of network gossip peers", + )?, registry)?, + ready_transactions_number: register(Gauge::new( + "ready_transactions_number", "Number of transactions in the ready queue", + )?, registry)?, + memory_usage_bytes: register(Gauge::new( + "memory_usage_bytes", "Node memory usage", + )?, registry)?, + cpu_usage_percentage: register(Gauge::new( + "cpu_usage_percentage", "Node CPU usage", + )?, registry)?, + network_per_sec_bytes: register(GaugeVec::new( + Opts::new("network_per_sec_bytes", "Networking bytes per second"), + &["direction"] + )?, registry)?, + }) + } +} + +pub type BackgroundTask = Pin + Send>>; /// Aggregator for the components required to build a service. /// @@ -63,7 +102,6 @@ use grafana_data_source::{self, record_metrics}; /// /// - [`with_select_chain`](ServiceBuilder::with_select_chain) /// - [`with_import_queue`](ServiceBuilder::with_import_queue) -/// - [`with_network_protocol`](ServiceBuilder::with_network_protocol) /// - [`with_finality_proof_provider`](ServiceBuilder::with_finality_proof_provider) /// - [`with_transaction_pool`](ServiceBuilder::with_transaction_pool) /// @@ -73,7 +111,7 @@ use grafana_data_source::{self, record_metrics}; /// generics is done when you call `build`. /// pub struct ServiceBuilder + TExPool, TRpc, Backend> { config: Configuration, pub (crate) client: Arc, @@ -84,11 +122,12 @@ pub struct ServiceBuilder, finality_proof_provider: Option, - network_protocol: TNetP, transaction_pool: Arc, rpc_extensions: TRpc, remote_backend: Option>>, marker: PhantomData<(TBl, TRtApi)>, + background_tasks: Vec<(&'static str, BackgroundTask)>, + prometheus_registry: Option } /// Full client type. @@ -225,7 +264,7 @@ fn new_full_parts( Ok((client, backend, keystore)) } -impl ServiceBuilder<(), (), TGen, TCSExt, (), (), (), (), (), (), (), (), (), ()> +impl ServiceBuilder<(), (), TGen, TCSExt, (), (), (), (), (), (), (), (), ()> where TGen: RuntimeGenesis, TCSExt: Extension { /// Start the service builder with a configuration. pub fn new_full( @@ -243,7 +282,6 @@ where TGen: RuntimeGenesis, TCSExt: Extension { Arc>, (), (), - (), TFullBackend, >, Error> { let (client, backend, keystore) = new_full_parts(&config)?; @@ -260,11 +298,12 @@ where TGen: RuntimeGenesis, TCSExt: Extension { import_queue: (), finality_proof_request_builder: None, finality_proof_provider: None, - network_protocol: (), transaction_pool: Arc::new(()), rpc_extensions: Default::default(), remote_backend: None, + background_tasks: Default::default(), marker: PhantomData, + prometheus_registry: None, }) } @@ -284,7 +323,6 @@ where TGen: RuntimeGenesis, TCSExt: Extension { Arc>, (), (), - (), TLightBackend, >, Error> { let keystore = match &config.keystore { @@ -326,7 +364,7 @@ where TGen: RuntimeGenesis, TCSExt: Extension { executor.clone(), ), ); - let fetcher = Arc::new(sc_network::OnDemand::new(fetch_checker)); + let fetcher = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); let backend = sc_client::light::new_light_backend(light_blockchain); let remote_blockchain = backend.remote_blockchain(); let client = Arc::new(sc_client::light::new_light( @@ -345,18 +383,19 @@ where TGen: RuntimeGenesis, TCSExt: Extension { import_queue: (), finality_proof_request_builder: None, finality_proof_provider: None, - network_protocol: (), transaction_pool: Arc::new(()), rpc_extensions: Default::default(), remote_backend: Some(remote_blockchain), + background_tasks: Default::default(), marker: PhantomData, + prometheus_registry: None, }) } } -impl +impl ServiceBuilder { + TExPool, TRpc, Backend> { /// Returns a reference to the client that was stored in this builder. pub fn client(&self) -> &Arc { @@ -373,6 +412,30 @@ impl Arc> { + self.keystore.clone() + } + + /// Returns a reference to the transaction pool stored in this builder + pub fn pool(&self) -> Arc { + self.transaction_pool.clone() + } + + /// Returns a reference to the fetcher, only available if builder + /// was created with `new_light`. + pub fn fetcher(&self) -> Option + where TFchr: Clone + { + self.fetcher.clone() + } + + /// Returns a reference to the remote_backend, only available if builder + /// was created with `new_light`. + pub fn remote_backend(&self) -> Option>> { + self.remote_backend.clone() + } + /// Defines which head-of-chain strategy to use. pub fn with_opt_select_chain( self, @@ -380,7 +443,7 @@ impl, &Arc ) -> Result, Error> ) -> Result, Error> { + TExPool, TRpc, Backend>, Error> { let select_chain = select_chain_builder(&self.config, &self.backend)?; Ok(ServiceBuilder { @@ -393,11 +456,12 @@ impl, &Arc) -> Result ) -> Result, Error> { + TExPool, TRpc, Backend>, Error> { self.with_opt_select_chain(|cfg, b| builder(cfg, b).map(Option::Some)) } @@ -416,7 +480,7 @@ impl, Arc, Option, Arc) -> Result ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone { let import_queue = builder( &self.config, @@ -435,37 +499,12 @@ impl( - self, - network_protocol_builder: impl FnOnce(&Configuration) -> Result - ) -> Result, Error> { - let network_protocol = network_protocol_builder(&self.config)?; - - Ok(ServiceBuilder { - config: self.config, - client: self.client, - backend: self.backend, - keystore: self.keystore, - fetcher: self.fetcher, - select_chain: self.select_chain, - import_queue: self.import_queue, - finality_proof_request_builder: self.finality_proof_request_builder, - finality_proof_provider: self.finality_proof_provider, - network_protocol, transaction_pool: self.transaction_pool, rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, + background_tasks: self.background_tasks, marker: self.marker, + prometheus_registry: self.prometheus_registry, }) } @@ -484,7 +523,6 @@ impl>, - TNetP, TExPool, TRpc, Backend, @@ -501,11 +539,12 @@ impl>, - TNetP, TExPool, TRpc, Backend, @@ -544,7 +582,7 @@ impl, ) -> Result<(UImpQu, Option), Error> ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { let (import_queue, fprb) = builder( &self.config, @@ -565,11 +603,12 @@ impl, ) -> Result<(UImpQu, UFprb), Error> ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { self.with_import_queue_and_opt_fprb(|cfg, cl, b, f, sc, tx| builder(cfg, cl, b, f, sc, tx) @@ -595,21 +634,25 @@ impl( - self, + mut self, transaction_pool_builder: impl FnOnce( sc_transaction_pool::txpool::Options, Arc, Option, - ) -> Result + ) -> Result<(UExPool, Option), Error> ) -> Result, Error> + UExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { - let transaction_pool = transaction_pool_builder( + let (transaction_pool, background_task) = transaction_pool_builder( self.config.transaction_pool.clone(), self.client.clone(), self.fetcher.clone(), )?; + if let Some(background_task) = background_task{ + self.background_tasks.push(("txpool-background", background_task)); + } + Ok(ServiceBuilder { config: self.config, client: self.client, @@ -620,34 +663,23 @@ impl( self, - rpc_ext_builder: impl FnOnce( - Arc, - Arc, - Arc, - Option, - Option>>, - ) -> Result, + rpc_ext_builder: impl FnOnce(&Self) -> Result, ) -> Result, Error> + TExPool, URpc, Backend>, Error> where TSc: Clone, TFchr: Clone { - let rpc_extensions = rpc_ext_builder( - self.client.clone(), - self.transaction_pool.clone(), - self.backend.clone(), - self.fetcher.clone(), - self.remote_backend.clone(), - )?; + let rpc_extensions = rpc_ext_builder(&self)?; Ok(ServiceBuilder { config: self.config, @@ -659,13 +691,35 @@ impl Self { + Self { + config: self.config, + client: self.client, + backend: self.backend, + keystore: self.keystore, + fetcher: self.fetcher, + select_chain: self.select_chain, + import_queue: self.import_queue, + finality_proof_request_builder: self.finality_proof_request_builder, + finality_proof_provider: self.finality_proof_provider, + transaction_pool: self.transaction_pool, + rpc_extensions: self.rpc_extensions, + remote_backend: self.remote_backend, + background_tasks: self.background_tasks, + marker: self.marker, + prometheus_registry: Some(registry), + } + } } /// Implemented on `ServiceBuilder`. Allows running block commands, such as import/export/validate @@ -673,6 +727,8 @@ impl Pin> + Send>>; } -impl +impl ServiceBuilder< TBl, TRtApi, @@ -714,7 +770,6 @@ ServiceBuilder< TImpQu, BoxFinalityProofRequestBuilder, Arc>, - TNetP, TExPool, TRpc, TBackend, @@ -735,8 +790,7 @@ ServiceBuilder< TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, TSc: Clone, TImpQu: 'static + ImportQueue, - TNetP: NetworkSpecialization, - TExPool: MaintainedTransactionPool::Hash> + 'static, + TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, TRpc: sc_rpc::RpcExtension + Clone, { @@ -752,7 +806,7 @@ ServiceBuilder< Client, TSc, NetworkStatus, - NetworkService::Hash>, + NetworkService::Hash>, TExPool, sc_offchain::OffchainWorkers< Client, @@ -771,10 +825,11 @@ ServiceBuilder< import_queue, finality_proof_request_builder, finality_proof_provider, - network_protocol, transaction_pool, rpc_extensions, remote_backend, + background_tasks, + prometheus_registry, } = self; sp_session::generate_initial_session_keys( @@ -850,7 +905,6 @@ ServiceBuilder< transaction_pool: transaction_pool_adapter.clone() as _, import_queue, protocol_id, - specialization: network_protocol, block_announce_validator, }; @@ -871,6 +925,15 @@ ServiceBuilder< _ => None, }; + // Spawn background tasks which were stacked during the + // service building. + for (title, background_task) in background_tasks { + let _ = to_spawn_tx.unbounded_send(( + background_task, + title.into(), + )); + } + { // block notifications let txpool = Arc::downgrade(&transaction_pool); @@ -879,39 +942,58 @@ ServiceBuilder< let network_state_info: Arc = network.clone(); let is_validator = config.roles.is_authority(); - let events = client.import_notification_stream() - .for_each(move |notification| { - let txpool = txpool.upgrade(); + let (import_stream, finality_stream) = ( + client.import_notification_stream().map(|n| ChainEvent::NewBlock { + id: BlockId::Hash(n.hash), + header: n.header, + retracted: n.retracted, + is_new_best: n.is_new_best, + }), + client.finality_notification_stream().map(|n| ChainEvent::Finalized { + hash: n.hash + }) + ); + let events = futures::stream::select(import_stream, finality_stream) + .for_each(move |event| { + // offchain worker is only interested in block import events + if let ChainEvent::NewBlock { ref header, is_new_best, .. } = event { + let offchain = offchain.as_ref().and_then(|o| o.upgrade()); + match offchain { + Some(offchain) if is_new_best => { + let future = offchain.on_block_imported( + &header, + network_state_info.clone(), + is_validator, + ); + let _ = to_spawn_tx_.unbounded_send(( + Box::pin(future), + From::from("offchain-on-block"), + )); + }, + Some(_) => log::debug!( + target: "sc_offchain", + "Skipping offchain workers for non-canon block: {:?}", + header, + ), + _ => {}, + } + }; + let txpool = txpool.upgrade(); if let Some(txpool) = txpool.as_ref() { - let future = txpool.maintain( - &BlockId::hash(notification.hash), - ¬ification.retracted, - ); + let future = txpool.maintain(event); let _ = to_spawn_tx_.unbounded_send(( Box::pin(future), From::from("txpool-maintain") )); } - let offchain = offchain.as_ref().and_then(|o| o.upgrade()); - if let Some(offchain) = offchain { - let future = offchain.on_block_imported( - ¬ification.header, - network_state_info.clone(), - is_validator - ); - let _ = to_spawn_tx_.unbounded_send(( - Box::pin(future), - From::from("offchain-on-block") - )); - } - ready(()) }); + let _ = to_spawn_tx.unbounded_send(( Box::pin(select(events, exit.clone()).map(drop)), - From::from("txpool-and-offchain-notif") + From::from("txpool-and-offchain-notif"), )); } @@ -934,10 +1016,34 @@ ServiceBuilder< let _ = to_spawn_tx.unbounded_send(( Box::pin(select(events, exit.clone()).map(drop)), - From::from("telemetry-on-block") + From::from("telemetry-on-block"), )); } + // Prometheus exporter and metrics + let metrics = if let Some(port) = config.prometheus_port { + let registry = match prometheus_registry { + Some(registry) => registry, + None => Registry::new_custom(Some("substrate".into()), None)? + }; + + let metrics = ServiceMetrics::register(®istry)?; + + let future = select( + prometheus_exporter::init_prometheus(port, registry).boxed(), + exit.clone() + ).map(drop); + + let _ = to_spawn_tx.unbounded_send(( + Box::pin(future), + From::from("prometheus-endpoint") + )); + + Some(metrics) + } else { + None + }; + // Periodically notify the telemetry. let transaction_pool_ = transaction_pool.clone(); let client_ = client.clone(); @@ -954,6 +1060,8 @@ ServiceBuilder< let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); let bandwidth_download = net_status.average_download_per_sec; let bandwidth_upload = net_status.average_upload_per_sec; + let best_seen_block = net_status.best_seen_block + .map(|num: NumberFor| num.unique_saturated_into() as u64); // get cpu usage and memory usage of this process let (cpu_usage, memory) = if let Some(self_pid) = self_pid { @@ -982,28 +1090,29 @@ ServiceBuilder< "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), ); - let _ = record_metrics!( - "peers" => num_peers, - "height" => best_number, - "txcount" => txpool_status.ready, - "cpu" => cpu_usage, - "memory" => memory, - "finalized_height" => finalized_number, - "bandwidth_download" => bandwidth_download, - "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0), - "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), - "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), - "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), - "memory_transaction_pool" => parity_util_mem::malloc_size(&*transaction_pool_), - ); + if let Some(metrics) = metrics.as_ref() { + metrics.memory_usage_bytes.set(memory); + metrics.cpu_usage_percentage.set(f64::from(cpu_usage)); + metrics.ready_transactions_number.set(txpool_status.ready as u64); + metrics.peers_count.set(num_peers as u64); + + metrics.network_per_sec_bytes.with_label_values(&["download"]).set(net_status.average_download_per_sec); + metrics.network_per_sec_bytes.with_label_values(&["upload"]).set(net_status.average_upload_per_sec); + + metrics.block_height_number.with_label_values(&["finalized"]).set(finalized_number); + metrics.block_height_number.with_label_values(&["best"]).set(best_number); + + if let Some(best_seen_block) = best_seen_block { + metrics.block_height_number.with_label_values(&["sync_target"]).set(best_seen_block); + } + } ready(()) }); let _ = to_spawn_tx.unbounded_send(( Box::pin(select(tel_task, exit.clone()).map(drop)), - From::from("telemetry-periodic-send") + From::from("telemetry-periodic-send"), )); // Periodically send the network state to the telemetry. @@ -1019,7 +1128,7 @@ ServiceBuilder< }); let _ = to_spawn_tx.unbounded_send(( Box::pin(select(tel_task_2, exit.clone()).map(drop)), - From::from("telemetry-periodic-network-state") + From::from("telemetry-periodic-network-state"), )); // RPC @@ -1105,7 +1214,7 @@ ServiceBuilder< system_rpc_rx, has_bootnodes, ), exit.clone()).map(drop)), - From::from("network-worker") + From::from("network-worker"), )); let telemetry_connection_sinks: Arc>>> = Default::default(); @@ -1153,16 +1262,6 @@ ServiceBuilder< telemetry }); - // Grafana data source - if let Some(port) = config.grafana_port { - let future = select( - grafana_data_source::run_server(port).boxed(), - exit.clone() - ).map(drop); - - let _ = to_spawn_tx.unbounded_send((Box::pin(future), From::from("grafana-server"))); - } - // Instrumentation if let Some(tracing_targets) = config.tracing_targets.as_ref() { let subscriber = sc_tracing::ProfilingSubscriber::new( diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index cac1c14d0b45b0d23b9b973c8a2dad8cdcf91eb5..a0724f3e1decc60453e8ff704b4c331dc6f005f9 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -27,14 +27,15 @@ use sp_runtime::traits::{ }; use sp_runtime::generic::{BlockId, SignedBlock}; use codec::{Decode, Encode, IoReader}; -use sc_client::Client; -use sp_consensus::import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}; -use sp_consensus::BlockOrigin; +use sc_client::{Client, LocalCallExecutor}; +use sp_consensus::{ + BlockOrigin, + import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}, +}; +use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use std::{io::{Read, Write, Seek}, pin::Pin}; -use sc_network::message; - /// Build a chain spec json pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result where G: RuntimeGenesis, @@ -45,19 +46,21 @@ pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result ServiceBuilderCommand for ServiceBuilder< - TBl, TRtApi, TGen, TCSExt, Client, - TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TExPool, TRpc, Backend + TBl, TRtApi, TGen, TCSExt, + Client>, TBl, TRtApi>, + TFchr, TSc, TImpQu, TFprb, TFpp, TExPool, TRpc, Backend > where TBl: BlockT, TBackend: 'static + sc_client_api::backend::Backend + Send, - TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, + TExecDisp: 'static + NativeExecutionDispatch, TImpQu: 'static + ImportQueue, TRtApi: 'static + Send + Sync, { type Block = TBl; + type NativeDispatch = TExecDisp; fn import_blocks( self, @@ -136,21 +139,13 @@ impl< Ok(signed) => { let (header, extrinsics) = signed.block.deconstruct(); let hash = header.hash(); - let block = message::BlockData:: { - hash, - justification: signed.justification, - header: Some(header), - body: Some(extrinsics), - receipt: None, - message_queue: None - }; // import queue handles verification and importing it into the client queue.import_blocks(BlockOrigin::File, vec![ IncomingBlock:: { - hash: block.hash, - header: block.header, - body: block.body, - justification: block.justification, + hash, + header: Some(header), + body: Some(extrinsics), + justification: signed.justification, origin: None, allow_missing_state: false, import_existing: force, diff --git a/client/service/src/config.rs b/client/service/src/config.rs index f4043d533e190bb150be44db46763f7edcede498..002754f11ce932168aaaaaf6b37dbe9ba51d84a6 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -93,8 +93,8 @@ pub struct Configuration { pub rpc_ws_max_connections: Option, /// CORS settings for HTTP & WS servers. `None` if all origins are allowed. pub rpc_cors: Option>, - /// Grafana data source http port. `None` if disabled. - pub grafana_port: Option, + /// Prometheus exporter Port. `None` if disabled. + pub prometheus_port: Option, /// Telemetry service URL. `None` if disabled. pub telemetry_endpoints: Option, /// External WASM transport for the telemetry. If `Some`, when connection to a telemetry @@ -137,7 +137,7 @@ pub enum KeystoreConfig { password: Option> }, /// In-memory keystore. Recommended for in-browser nodes. - InMemory + InMemory, } impl KeystoreConfig { @@ -190,7 +190,7 @@ impl Default for Configuration { rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: Some(vec![]), - grafana_port: None, + prometheus_port: None, telemetry_endpoints: None, telemetry_external_transport: None, default_heap_pages: None, @@ -207,7 +207,7 @@ impl Default for Configuration { impl Configuration { /// Create a default config using `VersionInfo` - pub fn new(version: &VersionInfo) -> Self { + pub fn from_version(version: &VersionInfo) -> Self { let mut config = Configuration::default(); config.impl_name = version.name; config.impl_version = version.version; @@ -254,6 +254,28 @@ impl Configuration { pub fn expect_database(&self) -> &DatabaseConfig { self.database.as_ref().expect("database must be specified") } + + /// Returns a string displaying the node role, special casing the sentry mode + /// (returning `SENTRY`), since the node technically has an `AUTHORITY` role but + /// doesn't participate. + pub fn display_role(&self) -> String { + if self.sentry_mode { + "SENTRY".to_string() + } else { + self.roles.to_string() + } + } + + /// Use in memory keystore config when it is not required at all. + /// + /// This function returns an error if the keystore is already set to something different than + /// `KeystoreConfig::None`. + pub fn use_in_memory_keystore(&mut self) -> Result<(), String> { + match &mut self.keystore { + cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, + _ => Err("Keystore config specified when it should not be!".into()), + } + } } /// Returns platform info diff --git a/client/service/src/error.rs b/client/service/src/error.rs index 059e1c19e490d6861339e278a274f5c060382950..4d0a2cef942337a816de670882e7a6fe42df5d0e 100644 --- a/client/service/src/error.rs +++ b/client/service/src/error.rs @@ -53,6 +53,12 @@ impl<'a> From<&'a str> for Error { } } +impl From for Error { + fn from(e: prometheus_exporter::PrometheusError) -> Self { + Error::Other(format!("Prometheus error: {}", e)) + } +} + impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 577f36572acbbf2d7dc3c8c8fbff6dffc00d8117..99b45453411d3164c49778204f069b5191652eb8 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -31,7 +31,8 @@ use std::{borrow::Cow, io, pin::Pin}; use std::marker::PhantomData; use std::net::SocketAddr; use std::collections::HashMap; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; use std::task::{Poll, Context}; use parking_lot::Mutex; @@ -44,14 +45,12 @@ use futures::{ sink::SinkExt, task::{Spawn, FutureObj, SpawnError}, }; -use sc_network::{ - NetworkService, NetworkState, specialization::NetworkSpecialization, - PeerId, ReportHandle, -}; +use sc_network::{NetworkService, network_state::NetworkState, PeerId, ReportHandle}; use log::{log, warn, debug, error, Level}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT}; +use parity_util_mem::MallocSizeOf; pub use self::error::Error; pub use self::builder::{ @@ -65,13 +64,24 @@ pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoo pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; pub use sc_client::FinalityNotifications; pub use sc_rpc::Metadata as RpcMetadata; +pub use sc_executor::NativeExecutionDispatch; #[doc(hidden)] pub use std::{ops::Deref, result::Result, sync::Arc}; #[doc(hidden)] -pub use sc_network::{FinalityProofProvider, OnDemand, config::BoxFinalityProofRequestBuilder}; +pub use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; const DEFAULT_PROTOCOL_ID: &str = "sup"; +/// A type that implements `MallocSizeOf` on native but not wasm. +#[cfg(not(target_os = "unknown"))] +pub trait MallocSizeOfWasm: MallocSizeOf {} +#[cfg(target_os = "unknown")] +pub trait MallocSizeOfWasm {} +#[cfg(not(target_os = "unknown"))] +impl MallocSizeOfWasm for T {} +#[cfg(target_os = "unknown")] +impl MallocSizeOfWasm for T {} + /// Substrate service. pub struct Service { client: Arc, @@ -162,9 +172,7 @@ pub trait AbstractService: 'static + Future> + /// Chain selection algorithm. type SelectChain: sp_consensus::SelectChain; /// Transaction pool. - type TransactionPool: TransactionPool; - /// Network specialization. - type NetworkSpecialization: NetworkSpecialization; + type TransactionPool: TransactionPool + MallocSizeOfWasm; /// Get event stream for telemetry connection established events. fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()>; @@ -205,7 +213,7 @@ pub trait AbstractService: 'static + Future> + /// Get shared network instance. fn network(&self) - -> Arc::Hash>>; + -> Arc::Hash>>; /// Returns a receiver that periodically receives a status of the network. fn network_status(&self, interval: Duration) -> mpsc::UnboundedReceiver<(NetworkStatus, NetworkState)>; @@ -217,18 +225,17 @@ pub trait AbstractService: 'static + Future> + fn on_exit(&self) -> ::exit_future::Exit; } -impl AbstractService for +impl AbstractService for Service, TSc, NetworkStatus, - NetworkService, TExPool, TOc> + NetworkService, TExPool, TOc> where TBl: BlockT + Unpin, TBackend: 'static + sc_client_api::backend::Backend, TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, TRtApi: 'static + Send + Sync, TSc: sp_consensus::SelectChain + 'static + Clone + Send + Unpin, - TExPool: 'static + TransactionPool, + TExPool: 'static + TransactionPool + MallocSizeOfWasm, TOc: 'static + Send + Sync, - TNetSpec: NetworkSpecialization, { type Block = TBl; type Backend = TBackend; @@ -236,7 +243,6 @@ where type RuntimeApi = TRtApi; type SelectChain = TSc; type TransactionPool = TExPool; - type NetworkSpecialization = TNetSpec; fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()> { let (sink, stream) = futures::channel::mpsc::unbounded(); @@ -302,7 +308,7 @@ where } fn network(&self) - -> Arc::Hash>> + -> Arc::Hash>> { self.network.clone() } @@ -366,11 +372,10 @@ impl Spawn for fn build_network_future< B: BlockT, C: sc_client::BlockchainEvents, - S: sc_network::specialization::NetworkSpecialization, H: sc_network::ExHashT > ( roles: Roles, - mut network: sc_network::NetworkWorker, + mut network: sc_network::NetworkWorker, client: Arc, status_sinks: Arc, NetworkState)>>>, mut rpc_rx: mpsc::UnboundedReceiver>, @@ -384,7 +389,7 @@ fn build_network_future< // We poll `imported_blocks_stream`. while let Poll::Ready(Some(notification)) = Pin::new(&mut imported_blocks_stream).poll_next(cx) { - network.on_block_imported(notification.hash, notification.header, Vec::new(), notification.is_new_best); + network.on_block_imported(notification.header, Vec::new(), notification.is_new_best); } // We poll `finality_notification_stream`, but we only take the last event. @@ -521,6 +526,30 @@ impl Drop for } } +#[cfg(not(target_os = "unknown"))] +// Wrapper for HTTP and WS servers that makes sure they are properly shut down. +mod waiting { + pub struct HttpServer(pub Option); + impl Drop for HttpServer { + fn drop(&mut self) { + if let Some(server) = self.0.take() { + server.close_handle().close(); + server.wait(); + } + } + } + + pub struct WsServer(pub Option); + impl Drop for WsServer { + fn drop(&mut self) { + if let Some(server) = self.0.take() { + server.close_handle().close(); + let _ = server.wait(); + } + } + } +} + /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive. #[cfg(not(target_os = "unknown"))] fn start_rpc_servers sc_rpc_server::RpcHandler>( @@ -549,7 +578,7 @@ fn start_rpc_servers sc_rpc_server::RpcHandler sc_rpc_server::RpcHandler, { pool.ready() - .filter(|t| t.is_propagateable()) + .filter(|t| t.is_propagable()) .map(|t| { let hash = t.hash().clone(); let ex: B::Extrinsic = t.data().clone(); @@ -621,10 +650,10 @@ where .collect() } -impl sc_network::TransactionPool for +impl sc_network::config::TransactionPool for TransactionPoolAdapter where - C: sc_network::ClientHandle + Send + Sync, + C: sc_network::config::Client + Send + Sync, Pool: 'static + TransactionPool, B: BlockT, H: std::hash::Hash + Eq + sp_runtime::traits::Member + sp_runtime::traits::MaybeSerialize, @@ -692,6 +721,7 @@ mod tests { use futures::executor::block_on; use sp_consensus::SelectChain; use sp_runtime::traits::BlindCheckable; + use sp_runtime::generic::CheckSignature; use substrate_test_runtime_client::{prelude::*, runtime::{Extrinsic, Transfer}}; use sc_transaction_pool::{BasicPool, FullChainApi}; @@ -703,7 +733,7 @@ mod tests { let pool = Arc::new(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), - )); + ).0); let best = longest_chain.best_chain().unwrap(); let transaction = Transfer { amount: 5, @@ -720,7 +750,7 @@ mod tests { // then assert_eq!(transactions.len(), 1); - assert!(transactions[0].1.clone().check().is_ok()); + assert!(transactions[0].1.clone().check(CheckSignature::Yes).is_ok()); // this should not panic let _ = transactions[0].1.transfer(); } diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index c9dbe97464aa135a12dc2928eb2262478152315d..80683ab89e7bf7e686de7b3b0c76814447657ff6 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-service-test" -version = "2.0.0" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] tempfile = "3.1.0" @@ -13,10 +16,10 @@ log = "0.4.8" env_logger = "0.7.0" fdlimit = "0.1.1" futures = { version = "0.3.1", features = ["compat"] } -sc-service = { version = "0.8.0", default-features = false, path = "../../service" } -sc-network = { version = "0.8", path = "../../network" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8", path = "../../" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +sc-service = { version = "0.8.0-alpha.1", default-features = false, path = "../../service" } +sc-network = { version = "0.8.0-alpha.1", path = "../../network" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../../primitives/consensus/common" } +sc-client = { version = "0.8.0-alpha.1", path = "../../" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../primitives/transaction-pool" } diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 2976e66a2982f6190699736a002da3b712bd72fd..5f679b82b39a4896db0368951815fe13d846fe99 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -141,7 +141,7 @@ fn node_config ( { let root = root.path().join(format!("node-{}", index)); - let config_path = Some(String::from(root.join("network").to_str().unwrap())); + let config_path = Some(root.join("network")); let net_config_path = config_path.clone(); let network_config = NetworkConfiguration { @@ -166,6 +166,7 @@ fn node_config ( enable_mdns: false, allow_private_ipv4: true, wasm_external_transport: None, + use_yamux_flow_control: true, }, max_parallel_downloads: NetworkConfiguration::default().max_parallel_downloads, }; @@ -198,7 +199,7 @@ fn node_config ( rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: None, - grafana_port: None, + prometheus_port: None, telemetry_endpoints: None, telemetry_external_transport: None, default_heap_pages: None, @@ -497,7 +498,7 @@ pub fn consensus( const NUM_FULL_NODES: usize = 10; const NUM_LIGHT_NODES: usize = 10; const NUM_BLOCKS: usize = 10; // 10 * 2 sec block production time = ~20 seconds - let temp = tempdir_with_prefix("substrate-conensus-test"); + let temp = tempdir_with_prefix("substrate-consensus-test"); let mut network = TestNet::new( &temp, spec.clone(), diff --git a/client/src/cht.rs b/client/src/cht.rs index 29f19a77504b90fc5344cf12ec605e8dee267a0c..1435b77ec592e21d1a05be901662653dc6fcfbc1 100644 --- a/client/src/cht.rs +++ b/client/src/cht.rs @@ -28,7 +28,7 @@ use codec::Encode; use sp_trie; use sp_core::{H256, convert_hash}; -use sp_runtime::traits::{Header as HeaderT, SimpleArithmetic, Zero, One}; +use sp_runtime::traits::{Header as HeaderT, AtLeast32Bit, Zero, One}; use sp_state_machine::{ MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend @@ -48,7 +48,7 @@ pub fn size>() -> N { /// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized. pub fn is_build_required(cht_size: N, block_num: N) -> Option where - N: Clone + SimpleArithmetic, + N: Clone + AtLeast32Bit, { let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?; let two = N::one() + N::one(); @@ -66,7 +66,7 @@ pub fn is_build_required(cht_size: N, block_num: N) -> Option /// Returns Some(max_cht_number) if CHT has ever been built given maximal canonical block number. pub fn max_cht_number(cht_size: N, max_canonical_block: N) -> Option where - N: Clone + SimpleArithmetic, + N: Clone + AtLeast32Bit, { let max_cht_number = block_to_cht_number(cht_size, max_canonical_block)?; let two = N::one() + N::one(); @@ -291,18 +291,18 @@ fn build_pairs( /// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE). /// This is because the genesis hash is assumed to be known /// and including it would be redundant. -pub fn start_number(cht_size: N, cht_num: N) -> N { +pub fn start_number(cht_size: N, cht_num: N) -> N { (cht_num * cht_size) + N::one() } /// Get the ending block of a given CHT. -pub fn end_number(cht_size: N, cht_num: N) -> N { +pub fn end_number(cht_size: N, cht_num: N) -> N { (cht_num + N::one()) * cht_size } /// Convert a block number to a CHT number. /// Returns `None` for `block_num` == 0, `Some` otherwise. -pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { +pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { if block_num == N::zero() { None } else { diff --git a/client/src/client.rs b/client/src/client.rs index 9e30c7b2ea8afd606c4e59331f56cab36428eea2..e9a8f1228c5e19e15ba21d6702301342b623b2cf 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1192,7 +1192,7 @@ impl Client where // NOTE: we're setting the finalized block as best block, this might // be slightly inaccurate since we might have a "better" block // further along this chain, but since best chain selection logic is - // pluggable we cannot make a better choice here. usages that need + // plugable we cannot make a better choice here. usages that need // an accurate "best" block need to go through `SelectChain` // instead. operation.op.mark_head(BlockId::Hash(block))?; @@ -1599,6 +1599,9 @@ impl sp_consensus::BlockImport for &Client>, new_cache: HashMap>, ) -> Result { + let span = tracing::span!(tracing::Level::DEBUG, "import_block"); + let _enter = span.enter(); + if let Some(res) = self.prepare_block_storage_changes(&mut import_block).map_err(|e| { warn!("Block prepare storage changes error:\n{:?}", e); ConsensusError::ClientImport(e.to_string()) @@ -3038,7 +3041,7 @@ pub(crate) mod tests { .unwrap().build().unwrap().block; // we will finalize A2 which should make it impossible to import a new - // B3 at the same height but that doesnt't include it + // B3 at the same height but that doesn't include it ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap(); let import_err = client.import(BlockOrigin::Own, b3).err().unwrap(); diff --git a/client/src/leaves.rs b/client/src/leaves.rs index bb556da83a84a2f1bfb6a3f7b0ce2491c5926bc8..1082e6ca071eac8463f0c95163e4e8a86b1a6cb3 100644 --- a/client/src/leaves.rs +++ b/client/src/leaves.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::cmp::Reverse; use kvdb::{KeyValueDB, DBTransaction}; -use sp_runtime::traits::SimpleArithmetic; +use sp_runtime::traits::AtLeast32Bit; use codec::{Encode, Decode}; use sp_blockchain::{Error, Result}; @@ -65,7 +65,7 @@ pub struct LeafSet { impl LeafSet where H: Clone + PartialEq + Decode + Encode, - N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode, + N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode, { /// Construct a new, blank leaf set. pub fn new() -> Self { @@ -251,7 +251,7 @@ pub struct Undo<'a, H: 'a, N: 'a> { impl<'a, H: 'a, N: 'a> Undo<'a, H, N> where H: Clone + PartialEq + Decode + Encode, - N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode, + N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode, { /// Undo an imported block by providing the displaced leaf. pub fn undo_import(&mut self, displaced: ImportDisplaced) { diff --git a/client/src/lib.rs b/client/src/lib.rs index 4caabfa201fe6474d6ec31f806793cd1728cf563..d97246d478c2b6df0eb13128c4ceba23d9ccd6b0 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -104,4 +104,4 @@ pub use crate::{ }, leaves::LeafSet, }; -pub use sp_state_machine::{ExecutionStrategy, StorageProof}; +pub use sp_state_machine::{ExecutionStrategy, StorageProof, StateMachine}; diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index d66108b7f0adba6a9d414d3fc7d7dc9ff5f84bbb..9df6a38630684f8f545f2a4f1175fa3b21d386f6 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -25,7 +25,7 @@ use codec::{Decode, Encode}; use sp_core::{convert_hash, traits::CodeExecutor}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, - SimpleArithmetic, CheckedConversion, + AtLeast32Bit, CheckedConversion, }; use sp_state_machine::{ ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, @@ -286,7 +286,7 @@ impl FetchChecker for LightDataChecker } /// A view of BTreeMap as a changes trie roots storage. -struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> { +struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { roots: (Number, &'a [Hash]), prev_roots: &'a BTreeMap, } @@ -294,7 +294,7 @@ struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> { impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> where H: Hasher, - Number: ::std::fmt::Display + ::std::hash::Hash + Clone + SimpleArithmetic + Encode + Decode + Send + Sync + 'static, + Number: ::std::fmt::Display + ::std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, { fn build_anchor( diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index 1bfa13693780a5e549d8d1fa82c5436ab54e4c98..84ab26216290b59ef5f0eacf2fc14a5b986897d1 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -1,14 +1,16 @@ [package] name = "sc-state-db" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] parking_lot = "0.10.0" log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } [dev-dependencies] diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index f2722ae308068ea2f03720f3d16a1b3bdc9a6df3..f670e4f35f3ba2cd6bd7114fd88997afbf3f7ce4 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -102,7 +102,7 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Error::Db(e) => e.fmt(f), - Error::Decoding(e) => write!(f, "Error decoding slicable value: {}", e.what()), + Error::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e.what()), Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"), Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"), Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"), @@ -340,7 +340,7 @@ impl StateDbSync { { let refs = self.pinned.entry(hash.clone()).or_default(); if *refs == 0 { - trace!(target: "state-db", "Pinned block: {:?}", hash); + trace!(target: "state-db-pin", "Pinned block: {:?}", hash); self.non_canonical.pin(hash); } *refs += 1; @@ -357,11 +357,11 @@ impl StateDbSync { Entry::Occupied(mut entry) => { *entry.get_mut() -= 1; if *entry.get() == 0 { - trace!(target: "state-db", "Unpinned block: {:?}", hash); + trace!(target: "state-db-pin", "Unpinned block: {:?}", hash); entry.remove(); self.non_canonical.unpin(hash); } else { - trace!(target: "state-db", "Releasing reference for {:?}", hash); + trace!(target: "state-db-pin", "Releasing reference for {:?}", hash); } }, Entry::Vacant(_) => {}, diff --git a/client/state-db/src/noncanonical.rs b/client/state-db/src/noncanonical.rs index 373c1aa0da076690ff29299f1286efb887851b36..de7294d770a43d9f4e534132b55b14a1d5a68650 100644 --- a/client/state-db/src/noncanonical.rs +++ b/client/state-db/src/noncanonical.rs @@ -106,7 +106,7 @@ fn discard_descendants( // save to be discarded later. pinned_insertions.insert(overlay.hash.clone(), overlay.inserted); } else { - // discard immediatelly. + // discard immediately. parents.remove(&overlay.hash); discard_values(&mut values, overlay.inserted); } @@ -436,7 +436,7 @@ impl NonCanonicalOverlay { while let Some(hash) = parent { let refs = self.pinned.entry(hash.clone()).or_default(); if *refs == 0 { - trace!(target: "state-db", "Pinned non-canon block: {:?}", hash); + trace!(target: "state-db-pin", "Pinned non-canon block: {:?}", hash); } *refs += 1; parent = self.parents.get(hash); @@ -455,7 +455,7 @@ impl NonCanonicalOverlay { if *entry.get() == 0 { entry.remove(); if let Some(inserted) = self.pinned_insertions.remove(&hash) { - trace!(target: "state-db", "Discarding unpinned non-canon block: {:?}", hash); + trace!(target: "state-db-pin", "Discarding unpinned non-canon block: {:?}", hash); discard_values(&mut self.values, inserted); self.parents.remove(&hash); } diff --git a/client/state-db/src/pruning.rs b/client/state-db/src/pruning.rs index a993df4f111ac756828c2624c576af1c8b4a1b5f..71d018087b5903cfdb8ccc24999612fcfb12c02f 100644 --- a/client/state-db/src/pruning.rs +++ b/client/state-db/src/pruning.rs @@ -36,7 +36,7 @@ pub struct RefWindow { death_rows: VecDeque>, /// An index that maps each key from `death_rows` to block number. death_index: HashMap, - /// Block number that corresponts to the front of `death_rows` + /// Block number that corresponds to the front of `death_rows`. pending_number: u64, /// Number of call of `note_canonical` after /// last call `apply_pending` or `revert_pending` @@ -348,7 +348,7 @@ mod tests { } #[test] - fn reinserted_survivew_pending() { + fn reinserted_survive_pending() { let mut db = make_db(&[1, 2, 3]); let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); let mut commit = make_commit(&[], &[2]); diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 44e332b9d7362bfc7119087425930061a0f47667..7ba8abafba5be2c10f2eb72b36df93afd381477d 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -1,17 +1,20 @@ [package] name = "sc-telemetry" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] description = "Telemetry utils" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] bytes = "0.5" parking_lot = "0.10.0" futures = "0.3.1" -futures-timer = "2.0.0" -libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } +futures-timer = "3.0.1" +wasm-timer = "0.2.0" +libp2p = { version = "0.16.1", default-features = false, features = ["libp2p-websocket"] } log = "0.4.8" pin-project = "0.4.6" rand = "0.7.2" diff --git a/client/telemetry/src/lib.rs b/client/telemetry/src/lib.rs index 906df545e2377484d8d116cfbf3abacc3b0c2806..f8ca6d5c73d41bdd86dff66a2ebbff8dc7d33dbc 100644 --- a/client/telemetry/src/lib.rs +++ b/client/telemetry/src/lib.rs @@ -63,7 +63,8 @@ use libp2p::{Multiaddr, wasm_ext}; use log::{error, warn}; use parking_lot::Mutex; use serde::{Serialize, Deserialize}; -use std::{pin::Pin, sync::Arc, task::{Context, Poll}, time::{Duration, Instant}}; +use std::{pin::Pin, sync::Arc, task::{Context, Poll}, time::Duration}; +use wasm_timer::Instant; pub use libp2p::wasm_ext::ExtTransport; pub use slog_scope::with_logger; @@ -125,7 +126,7 @@ pub struct Telemetry { /// Behind the `Mutex` in `Telemetry`. /// -/// Note that ideally we wouldn't have to make the `Telemetry` clonable, as that would remove the +/// Note that ideally we wouldn't have to make the `Telemetry` cloneable, as that would remove the /// need for a `Mutex`. However there is currently a weird hack in place in `sc-service` /// where we extract the telemetry registration so that it continues running during the shutdown /// process. @@ -194,7 +195,7 @@ impl Stream for Telemetry { fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let before = Instant::now(); - // Because the `Telemetry` is clonable, we need to put the actual fields behind a `Mutex`. + // Because the `Telemetry` is cloneable, we need to put the actual fields behind a `Mutex`. // However, the user is only ever supposed to poll from one instance of `Telemetry`, while // the other instances are used only for RAII purposes. // We assume that the user is following this advice and therefore that the `Mutex` is only diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index b476db6a0113d73167196b22f839ec2d0844e282..e508c728f3e63dede163c500e58fe0107b86248b 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-tracing" -version = "2.0.0" +version = "2.0.0-alpha.1" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] erased-serde = "0.3.9" @@ -14,8 +16,7 @@ serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } tracing-core = "0.1.7" -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -grafana-data-source = { version = "0.8", path = "../../utils/grafana-data-source" } +sc-telemetry = { version = "2.0.0-alpha.1", path = "../telemetry" } [dev-dependencies] tracing = "0.1.10" diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs index cd301041d39af4854522e3aa49608b93b3d28bd0..c00bca9275eec7fa2bf99050b6413c99f846bc78 100644 --- a/client/tracing/src/lib.rs +++ b/client/tracing/src/lib.rs @@ -34,7 +34,7 @@ //! let span = tracing::span!(tracing::Level::INFO, "my_span_name", my_number = 10, a_key = "a value"); //! let _guard = span.enter(); //! ``` -//! Currently we provide `Log` (default), `Telemetry` and `Grafana` variants for `Receiver` +//! Currently we provide `Log` (default), `Telemetry` variants for `Receiver` use std::collections::HashMap; use std::fmt; @@ -53,7 +53,6 @@ use tracing_core::{ subscriber::Subscriber }; -use grafana_data_source::{self, record_metrics}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; /// Used to configure how to receive the metrics @@ -63,8 +62,6 @@ pub enum TracingReceiver { Log, /// Output to telemetry Telemetry, - /// Output to Grafana - Grafana, } impl Default for TracingReceiver { @@ -255,7 +252,6 @@ impl ProfilingSubscriber { match self.receiver { TracingReceiver::Log => print_log(span_datum), TracingReceiver::Telemetry => send_telemetry(span_datum), - TracingReceiver::Grafana => send_grafana(span_datum), } } } @@ -291,9 +287,3 @@ fn send_telemetry(span_datum: SpanDatum) { ); } -fn send_grafana(span_datum: SpanDatum) { - let name = format!("{}::{}", span_datum.target, span_datum.name); - if let Err(e) = record_metrics!(&name => span_datum.overall_time.as_nanos(),) { - log::warn!("Unable to send metrics to grafana: {:?}", e); - } -} diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 524e9a98a0dbf172280a8337f267e940a6f18ea1..025fcec881855f6cbd2c96f502febd6f01e97ca8 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0" } @@ -12,16 +14,18 @@ futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-transaction-graph = { version = "2.0.0", path = "./graph" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +wasm-timer = "0.2" +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../primitives/runtime" } +sc-transaction-graph = { version = "2.0.0-alpha.1", path = "./graph" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../primitives/transaction-pool" } +sc-client-api = { version = "2.0.0-alpha.1", path = "../api" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../primitives/blockchain" } +futures-timer = "2.0" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-transaction-pool = { version = "2.0.0", path = "../../test-utils/runtime/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.1", path = "../../primitives/keyring" } +substrate-test-runtime-transaction-pool = { version = "2.0.0-dev", path = "../../test-utils/runtime/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index 2d3172fc915447e15757c32a9e4bd3f5a6b6fe21..ba290dc616821578210b45c9038e016659fdc580 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "sc-transaction-graph" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" @@ -11,15 +13,18 @@ futures = "0.3.1" log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +wasm-timer = "0.2" +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-alpha.1", path = "../../../primitives/transaction-pool" } parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } +linked-hash-map = "0.5.2" [dev-dependencies] assert_matches = "1.3.0" codec = { package = "parity-scale-codec", version = "1.0.0" } -substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } criterion = "0.3" [[bench]] diff --git a/client/transaction-pool/graph/benches/basics.rs b/client/transaction-pool/graph/benches/basics.rs index f3f67f1446f5cea37904f82382544a83062c6630..54bbe930b393bd37a94cb648272f1a66199cb691 100644 --- a/client/transaction-pool/graph/benches/basics.rs +++ b/client/transaction-pool/graph/benches/basics.rs @@ -135,8 +135,8 @@ fn bench_configured(pool: Pool, number: u64) { let res = block_on(futures::future::join_all(futures.into_iter())); assert!(res.iter().all(Result::is_ok)); - assert_eq!(pool.status().future, 0); - assert_eq!(pool.status().ready, number as usize); + assert_eq!(pool.validated_pool().status().future, 0); + assert_eq!(pool.validated_pool().status().ready, number as usize); // Prune all transactions. let block_num = 6; @@ -147,8 +147,8 @@ fn bench_configured(pool: Pool, number: u64) { )).expect("Prune failed"); // pool is empty - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); } fn benchmark_main(c: &mut Criterion) { diff --git a/client/transaction-pool/graph/src/base_pool.rs b/client/transaction-pool/graph/src/base_pool.rs index 52e00df36395db46e1ada04a69e37c659d7228b0..33b7a51f9415b5bc04a288e3a773e817178b4130 100644 --- a/client/transaction-pool/graph/src/base_pool.rs +++ b/client/transaction-pool/graph/src/base_pool.rs @@ -138,7 +138,7 @@ impl InPoolTransaction for Transaction { &self.provides } - fn is_propagateable(&self) -> bool { + fn is_propagable(&self) -> bool { self.propagate } } @@ -209,7 +209,8 @@ const RECENTLY_PRUNED_TAGS: usize = 2; /// as-is for the second time will fail or produce unwanted results. /// Most likely it is required to revalidate them and recompute set of /// required tags. -#[derive(Debug, parity_util_mem::MallocSizeOf)] +#[derive(Debug)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] pub struct BasePool { reject_future_transactions: bool, future: FutureTransactions, @@ -1057,7 +1058,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: true, - }.is_propagateable(), true); + }.is_propagable(), true); assert_eq!(Transaction { data: vec![4u8], @@ -1068,7 +1069,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() requires: vec![vec![3], vec![2]], provides: vec![vec![4]], propagate: false, - }.is_propagateable(), false); + }.is_propagable(), false); } #[test] @@ -1125,7 +1126,7 @@ requires: [03,02], provides: [04], data: [4]}".to_owned() } #[test] - fn should_accept_future_transactions_when_explcitly_asked_to() { + fn should_accept_future_transactions_when_explicitly_asked_to() { // given let mut pool = pool(); pool.reject_future_transactions = true; diff --git a/client/transaction-pool/graph/src/future.rs b/client/transaction-pool/graph/src/future.rs index bda26fe34f949a0ba6b26f2f8d8492f2eb5966cb..a84a5fbe689c37aacbba24ed1e6a8c6885961589 100644 --- a/client/transaction-pool/graph/src/future.rs +++ b/client/transaction-pool/graph/src/future.rs @@ -19,17 +19,17 @@ use std::{ fmt, hash, sync::Arc, - time, }; use sp_core::hexdisplay::HexDisplay; use sp_runtime::transaction_validity::{ TransactionTag as Tag, }; +use wasm_timer::Instant; use crate::base_pool::Transaction; -#[derive(parity_util_mem::MallocSizeOf)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] /// Transaction with partially satisfied dependencies. pub struct WaitingTransaction { /// Transaction details. @@ -37,7 +37,7 @@ pub struct WaitingTransaction { /// Tags that are required and have not been satisfied yet by other transactions in the pool. pub missing_tags: HashSet, /// Time of import to the Future Queue. - pub imported_at: time::Instant, + pub imported_at: Instant, } impl fmt::Debug for WaitingTransaction { @@ -91,7 +91,7 @@ impl WaitingTransaction { WaitingTransaction { transaction: Arc::new(transaction), missing_tags, - imported_at: time::Instant::now(), + imported_at: Instant::now(), } } @@ -110,7 +110,8 @@ impl WaitingTransaction { /// /// Contains transactions that are still awaiting for some other transactions that /// could provide a tag that they require. -#[derive(Debug, parity_util_mem::MallocSizeOf)] +#[derive(Debug)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] pub struct FutureTransactions { /// tags that are not yet provided by any transaction and we await for them wanted_tags: HashMap>, diff --git a/client/transaction-pool/graph/src/lib.rs b/client/transaction-pool/graph/src/lib.rs index 23970ba9b8f8c7014ab6d5b1e1c4811401ebdbd7..ed10ef38d2bfd868eda7504b74bbe3018dee0c6f 100644 --- a/client/transaction-pool/graph/src/lib.rs +++ b/client/transaction-pool/graph/src/lib.rs @@ -39,4 +39,5 @@ pub use self::pool::{ Pool, Options, ChainApi, EventStream, ExtrinsicFor, BlockHash, ExHash, NumberFor, TransactionFor, + ValidatedTransaction, }; diff --git a/client/transaction-pool/graph/src/listener.rs b/client/transaction-pool/graph/src/listener.rs index dab2a6f5aae3f5021b8cb4d71357db9257f32b64..92ba71007e209f6ef06d249864b25cdda191776b 100644 --- a/client/transaction-pool/graph/src/listener.rs +++ b/client/transaction-pool/graph/src/listener.rs @@ -16,30 +16,34 @@ // along with Substrate. If not, see . use std::{ - collections::HashMap, - fmt, - hash, + collections::HashMap, hash, }; +use linked_hash_map::LinkedHashMap; use serde::Serialize; -use crate::watcher; -use sp_runtime::traits; +use crate::{watcher, ChainApi, BlockHash}; use log::{debug, trace, warn}; +use sp_runtime::traits; /// Extrinsic pool default listener. -pub struct Listener { - watchers: HashMap> +pub struct Listener { + watchers: HashMap>>, + finality_watchers: LinkedHashMap, Vec>, } -impl Default for Listener { +/// Maximum number of blocks awaiting finality at any time. +const MAX_FINALITY_WATCHERS: usize = 512; + +impl Default for Listener { fn default() -> Self { Listener { watchers: Default::default(), + finality_watchers: Default::default(), } } } -impl Listener { - fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender) { +impl Listener { + fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender>) { let clean = if let Some(h) = self.watchers.get_mut(hash) { fun(h); h.is_done() @@ -54,8 +58,8 @@ impl Listene /// Creates a new watcher for given verified extrinsic. /// - /// The watcher can be used to subscribe to lifecycle events of that extrinsic. - pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher { + /// The watcher can be used to subscribe to life-cycle events of that extrinsic. + pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher> { let sender = self.watchers.entry(hash.clone()).or_insert_with(watcher::Sender::default); sender.new_watcher(hash) } @@ -101,8 +105,34 @@ impl Listene } /// Transaction was pruned from the pool. - pub fn pruned(&mut self, header_hash: H2, tx: &H) { - debug!(target: "txpool", "[{:?}] Pruned at {:?}", tx, header_hash); - self.fire(tx, |watcher| watcher.in_block(header_hash)) + pub fn pruned(&mut self, block_hash: BlockHash, tx: &H) { + debug!(target: "txpool", "[{:?}] Pruned at {:?}", tx, block_hash); + self.fire(tx, |s| s.in_block(block_hash)); + self.finality_watchers.entry(block_hash).or_insert(vec![]).push(tx.clone()); + + while self.finality_watchers.len() > MAX_FINALITY_WATCHERS { + if let Some((hash, txs)) = self.finality_watchers.pop_front() { + for tx in txs { + self.fire(&tx, |s| s.finality_timeout(hash.clone())); + } + } + } + } + + /// The block this transaction was included in has been retracted. + pub fn retracted(&mut self, block_hash: BlockHash) { + if let Some(hashes) = self.finality_watchers.remove(&block_hash) { + for hash in hashes { + self.fire(&hash, |s| s.retracted(block_hash)) + } + } + } + + /// Notify all watchers that transactions have been finalized + pub fn finalized(&mut self, block_hash: BlockHash, txs: Vec) { + self.finality_watchers.remove(&block_hash); + for h in txs { + self.fire(&h, |s| s.finalized(block_hash.clone())) + } } } diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index ab4e3a5a79f11bc31e627f06d2b93481f3fe5f0c..12601bb8d21be1e3349cf4eb2e6879215ac983a9 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -33,9 +33,11 @@ use sp_runtime::{ traits::{self, SaturatedConversion}, transaction_validity::{TransactionValidity, TransactionTag as Tag, TransactionValidityError}, }; -use sp_transaction_pool::{error, PoolStatus}; +use sp_transaction_pool::error; +use wasm_timer::Instant; -use crate::validated_pool::{ValidatedPool, ValidatedTransaction}; +use crate::validated_pool::ValidatedPool; +pub use crate::validated_pool::ValidatedTransaction; /// Modification notification event stream type; pub type EventStream = mpsc::UnboundedReceiver; @@ -122,6 +124,7 @@ pub struct Pool { validated_pool: Arc>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for Pool where B::Hash: parity_util_mem::MallocSizeOf, @@ -180,40 +183,19 @@ impl Pool { self.validated_pool.submit_and_watch(tx) } - /// Revalidate all ready transactions. - /// - /// Returns future that performs validation of all ready transactions and - /// then resubmits all transactions back to the pool. - pub async fn revalidate_ready( + /// Resubmit some transaction that were validated elsewhere. + pub fn resubmit( &self, - at: &BlockId, - max: Option, - ) -> Result<(), B::Error> { - use std::time::Instant; - log::debug!(target: "txpool", - "Fetching ready transactions (up to: {})", - max.map(|x| format!("{}", x)).unwrap_or_else(|| "all".into()) - ); - let validated_pool = self.validated_pool.clone(); - let ready = self.validated_pool.ready() - .map(|tx| tx.data.clone()) - .take(max.unwrap_or_else(usize::max_value)); - - let now = Instant::now(); - let revalidated_transactions = self.verify(at, ready, false).await?; - log::debug!(target: "txpool", - "Re-verified transactions, took {} ms. Resubmitting.", - now.elapsed().as_millis() - ); + revalidated_transactions: HashMap, ValidatedTransactionFor>, + ) { let now = Instant::now(); self.validated_pool.resubmit(revalidated_transactions); log::debug!(target: "txpool", "Resubmitted. Took {} ms. Status: {:?}", now.elapsed().as_millis(), - validated_pool.status() + self.validated_pool.status() ); - Ok(()) } /// Prunes known ready transactions. @@ -317,7 +299,7 @@ impl Pool { // Make sure that we don't revalidate extrinsics that were part of the recently // imported block. This is especially important for UTXO-like chains cause the // inputs are pruned so such transaction would go to future again. - self.validated_pool.ban(&std::time::Instant::now(), known_imported_hashes.clone().into_iter()); + self.validated_pool.ban(&Instant::now(), known_imported_hashes.clone().into_iter()); // Try to re-validate pruned transactions since some of them might be still valid. // note that `known_imported_hashes` will be rejected here due to temporary ban. @@ -326,7 +308,7 @@ impl Pool { let reverified_transactions = self.verify(at, pruned_transactions, false).await?; - log::trace!(target: "txpool", "Prunning at {:?}. Resubmitting transactions.", at); + log::trace!(target: "txpool", "Pruning at {:?}. Resubmitting transactions.", at); // And finally - submit reverified transactions back to the pool self.validated_pool.resubmit_pruned( @@ -337,34 +319,6 @@ impl Pool { ) } - /// Return an event stream of notifications for when transactions are imported to the pool. - /// - /// Consumers of this stream should use the `ready` method to actually get the - /// pending transactions in the right order. - pub fn import_notification_stream(&self) -> EventStream> { - self.validated_pool.import_notification_stream() - } - - /// Invoked when extrinsics are broadcasted. - pub fn on_broadcasted(&self, propagated: HashMap, Vec>) { - self.validated_pool.on_broadcasted(propagated) - } - - /// Remove invalid transactions from the pool. - pub fn remove_invalid(&self, hashes: &[ExHash]) -> Vec> { - self.validated_pool.remove_invalid(hashes) - } - - /// Get an iterator for ready transactions ordered by priority - pub fn ready(&self) -> impl Iterator> { - self.validated_pool.ready() - } - - /// Returns pool status. - pub fn status(&self) -> PoolStatus { - self.validated_pool.status() - } - /// Returns transaction hash pub fn hash_of(&self, xt: &ExtrinsicFor) -> ExHash { self.validated_pool.api().hash_and_length(xt).0 @@ -429,21 +383,15 @@ impl Pool { if validity.provides.is_empty() { ValidatedTransaction::Invalid(hash.clone(), error::Error::NoTagsProvided.into()) } else { - ValidatedTransaction::Valid(base::Transaction { - data: xt, + ValidatedTransaction::valid_at( + block_number.saturated_into::(), + hash.clone(), + xt, bytes, - hash: hash.clone(), - priority: validity.priority, - requires: validity.requires, - provides: validity.provides, - propagate: validity.propagate, - valid_till: block_number - .saturated_into::() - .saturating_add(validity.longevity), - }) + validity, + ) } }, - Err(TransactionValidityError::Invalid(e)) => ValidatedTransaction::Invalid(hash.clone(), error::Error::InvalidTransaction(e).into()), Err(TransactionValidityError::Unknown(e)) => @@ -453,9 +401,9 @@ impl Pool { (hash, validity) } - /// Get ready transaction by hash, if it present in the pool. - pub fn ready_transaction(&self, hash: &ExHash) -> Option> { - self.validated_pool.ready_by_hash(hash) + /// get a reference to the underlying validated pool. + pub fn validated_pool(&self) -> &ValidatedPool { + &self.validated_pool } } @@ -469,10 +417,7 @@ impl Clone for Pool { #[cfg(test)] mod tests { - use std::{ - collections::{HashMap, HashSet}, - time::Instant, - }; + use std::collections::{HashMap, HashSet}; use parking_lot::Mutex; use futures::executor::block_on; use super::*; @@ -481,6 +426,7 @@ mod tests { use codec::Encode; use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId}; use assert_matches::assert_matches; + use wasm_timer::Instant; use crate::base_pool::Limit; const INVALID_NONCE: u64 = 254; @@ -599,7 +545,7 @@ mod tests { }))).unwrap(); // then - assert_eq!(pool.ready().map(|v| v.hash).collect::>(), vec![hash]); + assert_eq!(pool.validated_pool().ready().map(|v| v.hash).collect::>(), vec![hash]); } #[test] @@ -616,8 +562,8 @@ mod tests { // when pool.validated_pool.rotator().ban(&Instant::now(), vec![pool.hash_of(&uxt)]); let res = block_on(pool.submit_one(&BlockId::Number(0), uxt)); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then assert_matches!(res.unwrap_err(), error::Error::TemporarilyBanned); @@ -628,7 +574,7 @@ mod tests { let stream = { // given let pool = pool(); - let stream = pool.import_notification_stream(); + let stream = pool.validated_pool().import_notification_stream(); // when let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -651,8 +597,8 @@ mod tests { nonce: 3, }))).unwrap(); - assert_eq!(pool.status().ready, 2); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().ready, 2); + assert_eq!(pool.validated_pool().status().future, 1); stream }; @@ -690,9 +636,9 @@ mod tests { pool.validated_pool.clear_stale(&BlockId::Number(5)).unwrap(); // then - assert_eq!(pool.ready().count(), 0); - assert_eq!(pool.status().future, 0); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().ready().count(), 0); + assert_eq!(pool.validated_pool().status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // make sure they are temporarily banned as well assert!(pool.validated_pool.rotator().is_banned(&hash1)); assert!(pool.validated_pool.rotator().is_banned(&hash2)); @@ -736,7 +682,7 @@ mod tests { amount: 5, nonce: 1, }))).unwrap(); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); // when let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -747,7 +693,7 @@ mod tests { }))).unwrap(); // then - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); assert!(pool.validated_pool.rotator().is_banned(&hash1)); assert!(!pool.validated_pool.rotator().is_banned(&hash2)); } @@ -774,8 +720,8 @@ mod tests { }))).unwrap_err(); // then - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); } #[test] @@ -792,8 +738,8 @@ mod tests { }))).unwrap_err(); // then - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); assert_matches!(err, error::Error::NoTagsProvided); } @@ -810,19 +756,18 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); // when block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![])).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); - assert_eq!(stream.next(), None); } #[test] @@ -835,19 +780,18 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); // when block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![2u64])).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); - assert_eq!(stream.next(), None); } #[test] @@ -860,8 +804,8 @@ mod tests { amount: 5, nonce: 1, }))).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 1); // when block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -870,7 +814,7 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 2); + assert_eq!(pool.validated_pool().status().ready, 2); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); @@ -889,7 +833,7 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when pool.validated_pool.remove_invalid(&[*watcher.hash()]); @@ -913,13 +857,13 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when let mut map = HashMap::new(); let peers = vec!["a".into(), "b".into(), "c".into()]; map.insert(*watcher.hash(), peers.clone()); - pool.on_broadcasted(map); + pool.validated_pool().on_broadcasted(map); // then @@ -948,7 +892,7 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when let xt = uxt(Transfer { @@ -958,7 +902,7 @@ mod tests { nonce: 1, }); block_on(pool.submit_one(&BlockId::Number(1), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); @@ -1001,11 +945,11 @@ mod tests { // The tag the above transaction provides (TestApi is using just nonce as u8) let provides = vec![0_u8]; block_on(pool.submit_one(&BlockId::Number(0), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // Now block import happens before the second transaction is able to finish verification. block_on(pool.prune_tags(&BlockId::Number(1), vec![provides], vec![])).unwrap(); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // so when we release the verification of the previous one it will have @@ -1015,85 +959,8 @@ mod tests { // then is_ready.recv().unwrap(); // wait for finish - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); } } - - #[test] - fn should_revalidate_ready_transactions() { - fn transfer(nonce: u64) -> Extrinsic { - uxt(Transfer { - from: AccountId::from_h256(H256::from_low_u64_be(1)), - to: AccountId::from_h256(H256::from_low_u64_be(2)), - amount: 5, - nonce, - }) - } - - // given - let pool = pool(); - let tx0 = transfer(0); - let hash0 = pool.validated_pool.api().hash_and_length(&tx0).0; - let watcher0 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx0)).unwrap(); - let tx1 = transfer(1); - let hash1 = pool.validated_pool.api().hash_and_length(&tx1).0; - let watcher1 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx1)).unwrap(); - let tx2 = transfer(2); - let hash2 = pool.validated_pool.api().hash_and_length(&tx2).0; - let watcher2 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx2)).unwrap(); - let tx3 = transfer(3); - let hash3 = pool.validated_pool.api().hash_and_length(&tx3).0; - let watcher3 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx3)).unwrap(); - let tx4 = transfer(4); - let hash4 = pool.validated_pool.api().hash_and_length(&tx4).0; - let watcher4 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx4)).unwrap(); - assert_eq!(pool.status().ready, 5); - - // when - pool.validated_pool.api().invalidate.lock().insert(hash3); - pool.validated_pool.api().clear_requirements.lock().insert(hash1); - pool.validated_pool.api().add_requirements.lock().insert(hash0); - block_on(pool.revalidate_ready(&BlockId::Number(0), None)).unwrap(); - - // then - // hash0 now has unsatisfied requirements => it is moved to the future queue - // hash1 is now independent of hash0 => it is in ready queue - // hash2 still depends on hash1 => it is in ready queue - // hash3 is now invalid => it is removed from the pool - // hash4 now depends on invalidated hash3 => it is moved to the future queue - // - // events for hash3 are: Ready, Invalid - // events for hash4 are: Ready, Invalid - assert_eq!(pool.status().ready, 2); - assert_eq!( - futures::executor::block_on_stream(watcher3.into_stream()).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::Invalid], - ); - - // when - pool.validated_pool.remove_invalid(&[hash0, hash1, hash2, hash4]); - - // then - // events for hash0 are: Ready, Future, Invalid - // events for hash1 are: Ready, Invalid - // events for hash2 are: Ready, Invalid - assert_eq!( - futures::executor::block_on_stream(watcher0.into_stream()).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::Future, TransactionStatus::Invalid], - ); - assert_eq!( - futures::executor::block_on_stream(watcher1.into_stream()).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::Invalid], - ); - assert_eq!( - futures::executor::block_on_stream(watcher2.into_stream()).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::Invalid], - ); - assert_eq!( - futures::executor::block_on_stream(watcher4.into_stream()).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::Future, TransactionStatus::Invalid], - ); - } } - diff --git a/client/transaction-pool/graph/src/rotator.rs b/client/transaction-pool/graph/src/rotator.rs index e1c852f95a418146c7b0eefc5b2d0804f5bb507d..55a9230522e2f7ee19e31cd004184d691ee8fd9e 100644 --- a/client/transaction-pool/graph/src/rotator.rs +++ b/client/transaction-pool/graph/src/rotator.rs @@ -23,9 +23,10 @@ use std::{ collections::HashMap, hash, iter, - time::{Duration, Instant}, + time::Duration, }; use parking_lot::RwLock; +use wasm_timer::Instant; use crate::base_pool::Transaction; diff --git a/client/transaction-pool/graph/src/validated_pool.rs b/client/transaction-pool/graph/src/validated_pool.rs index 95242840646c44dfe3daffccd012d6e3103f05ba..a62822a91858b3d0bc0023832e0fe82a880116c3 100644 --- a/client/transaction-pool/graph/src/validated_pool.rs +++ b/client/transaction-pool/graph/src/validated_pool.rs @@ -16,13 +16,11 @@ use std::{ collections::{HashSet, HashMap}, - fmt, hash, sync::Arc, - time, }; -use crate::base_pool as base; +use crate::{base_pool as base, BlockHash}; use crate::listener::Listener; use crate::rotator::PoolRotator; use crate::watcher::Watcher; @@ -34,12 +32,13 @@ use parking_lot::{Mutex, RwLock}; use sp_runtime::{ generic::BlockId, traits::{self, SaturatedConversion}, - transaction_validity::TransactionTag as Tag, + transaction_validity::{TransactionTag as Tag, ValidTransaction}, }; use sp_transaction_pool::{error, PoolStatus}; +use wasm_timer::Instant; use crate::base_pool::PruneStatus; -use crate::pool::{EventStream, Options, ChainApi, BlockHash, ExHash, ExtrinsicFor, TransactionFor}; +use crate::pool::{EventStream, Options, ChainApi, ExHash, ExtrinsicFor, TransactionFor}; /// Pre-validated transaction. Validated pool only accepts transactions wrapped in this enum. #[derive(Debug)] @@ -54,6 +53,30 @@ pub enum ValidatedTransaction { Unknown(Hash, Error), } +impl ValidatedTransaction { + /// Consume validity result, transaction data and produce ValidTransaction. + pub fn valid_at( + at: u64, + hash: Hash, + data: Ex, + bytes: usize, + validity: ValidTransaction, + ) -> Self { + Self::Valid(base::Transaction { + data, + bytes, + hash, + priority: validity.priority, + requires: validity.requires, + provides: validity.provides, + propagate: validity.propagate, + valid_till: at + .saturated_into::() + .saturating_add(validity.longevity), + }) + } +} + /// A type of validated transaction stored in the pool. pub type ValidatedTransactionFor = ValidatedTransaction< ExHash, @@ -62,10 +85,10 @@ pub type ValidatedTransactionFor = ValidatedTransaction< >; /// Pool that deals with validated transactions. -pub(crate) struct ValidatedPool { +pub struct ValidatedPool { api: Arc, options: Options, - listener: RwLock, BlockHash>>, + listener: RwLock, B>>, pool: RwLock, ExtrinsicFor, @@ -74,6 +97,7 @@ pub(crate) struct ValidatedPool { rotator: PoolRotator>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for ValidatedPool where B::Hash: parity_util_mem::MallocSizeOf, @@ -90,9 +114,9 @@ impl ValidatedPool { pub fn new(options: Options, api: Arc) -> Self { let base_pool = base::BasePool::new(options.reject_future_transactions); ValidatedPool { - api, options, listener: Default::default(), + api, pool: RwLock::new(base_pool), import_notification_sinks: Default::default(), rotator: Default::default(), @@ -100,7 +124,7 @@ impl ValidatedPool { } /// Bans given set of hashes. - pub fn ban(&self, now: &std::time::Instant, hashes: impl IntoIterator>) { + pub fn ban(&self, now: &Instant, hashes: impl IntoIterator>) { self.rotator.ban(now, hashes) } @@ -137,21 +161,22 @@ impl ValidatedPool { let imported = self.pool.write().import(tx)?; if let base::Imported::Ready { ref hash, .. } = imported { - self.import_notification_sinks.lock().retain(|sink| sink.unbounded_send(hash.clone()).is_ok()); + self.import_notification_sinks.lock() + .retain(|sink| sink.unbounded_send(hash.clone()).is_ok()); } let mut listener = self.listener.write(); fire_events(&mut *listener, &imported); Ok(imported.hash().clone()) - } + }, ValidatedTransaction::Invalid(hash, err) => { - self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash)); + self.rotator.ban(&Instant::now(), std::iter::once(hash)); Err(err.into()) }, ValidatedTransaction::Unknown(hash, err) => { self.listener.write().invalid(&hash, false); Err(err.into()) - } + }, } } @@ -177,7 +202,7 @@ impl ValidatedPool { let removed = pool.enforce_limits(ready_limit, future_limit) .into_iter().map(|x| x.hash.clone()).collect::>(); // ban all removed transactions - self.rotator.ban(&std::time::Instant::now(), removed.iter().map(|x| x.clone())); + self.rotator.ban(&Instant::now(), removed.iter().map(|x| x.clone())); removed }; // run notifications @@ -208,7 +233,7 @@ impl ValidatedPool { .map(|_| watcher) }, ValidatedTransaction::Invalid(hash, err) => { - self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash)); + self.rotator.ban(&Instant::now(), std::iter::once(hash)); Err(err.into()) }, ValidatedTransaction::Unknown(_, err) => Err(err.into()), @@ -342,8 +367,7 @@ impl ValidatedPool { self.pool.read().by_hashes(&hashes) .into_iter() .map(|existing_in_pool| existing_in_pool - .map(|transaction| transaction.provides.iter().cloned() - .collect())) + .map(|transaction| transaction.provides.iter().cloned().collect())) .collect() } @@ -415,8 +439,14 @@ impl ValidatedPool { let header_hash = self.api.block_id_to_hash(at)? .ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())?; let mut listener = self.listener.write(); + let mut set = HashSet::with_capacity(hashes.size_hint().0); for h in hashes { - listener.pruned(header_hash, &h); + // `hashes` has possibly duplicate hashes. + // we'd like to send out the `InBlock` notification only once. + if !set.contains(&h) { + listener.pruned(header_hash, &h); + set.insert(h); + } } Ok(()) } @@ -430,7 +460,7 @@ impl ValidatedPool { let block_number = self.api.block_id_to_number(at)? .ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())? .saturated_into::(); - let now = time::Instant::now(); + let now = Instant::now(); let to_remove = { self.ready() .filter(|tx| self.rotator.ban_if_stale(&now, block_number, &tx)) @@ -467,7 +497,10 @@ impl ValidatedPool { &self.api } - /// Return an event stream of transactions imported to the pool. + /// Return an event stream of notifications for when transactions are imported to the pool. + /// + /// Consumers of this stream should use the `ready` method to actually get the + /// pending transactions in the right order. pub fn import_notification_stream(&self) -> EventStream> { let (sink, stream) = mpsc::unbounded(); self.import_notification_sinks.lock().push(sink); @@ -491,13 +524,13 @@ impl ValidatedPool { pub fn remove_invalid(&self, hashes: &[ExHash]) -> Vec> { // early exit in case there is no invalid transactions. if hashes.is_empty() { - return vec![] + return vec![]; } debug!(target: "txpool", "Removing invalid transactions: {:?}", hashes); // temporarily ban invalid transactions - self.rotator.ban(&time::Instant::now(), hashes.iter().cloned()); + self.rotator.ban(&Instant::now(), hashes.iter().cloned()); let invalid = self.pool.write().remove_subtree(hashes); @@ -520,14 +553,34 @@ impl ValidatedPool { pub fn status(&self) -> PoolStatus { self.pool.read().status() } + + /// Notify all watchers that transactions in the block with hash have been finalized + pub async fn on_block_finalized(&self, block_hash: BlockHash) -> Result<(), B::Error> { + debug!(target: "txpool", "Attempting to notify watchers of finalization for {}", block_hash); + // fetch all extrinsic hashes + if let Some(txs) = self.api.block_body(&BlockId::Hash(block_hash.clone())).await? { + let tx_hashes = txs.into_iter() + .map(|tx| self.api.hash_and_length(&tx).0) + .collect::>(); + // notify the watcher that these extrinsics have been finalized + self.listener.write().finalized(block_hash, tx_hashes); + } + + Ok(()) + } + + /// Notify the listener of retracted blocks + pub fn on_block_retracted(&self, block_hash: BlockHash) { + self.listener.write().retracted(block_hash) + } } -fn fire_events( - listener: &mut Listener, +fn fire_events( + listener: &mut Listener, imported: &base::Imported, ) where H: hash::Hash + Eq + traits::Member + Serialize, - H2: Clone + fmt::Debug, + B: ChainApi, { match *imported { base::Imported::Ready { ref promoted, ref failed, ref removed, ref hash } => { diff --git a/client/transaction-pool/graph/src/watcher.rs b/client/transaction-pool/graph/src/watcher.rs index f9c234f73c3b8aad3fa86618750e9cab06b70676..d28f6814e455213a443b583646ee2af6a3abb335 100644 --- a/client/transaction-pool/graph/src/watcher.rs +++ b/client/transaction-pool/graph/src/watcher.rs @@ -26,12 +26,12 @@ use sp_transaction_pool::TransactionStatus; /// /// Represents a stream of status updates for particular extrinsic. #[derive(Debug)] -pub struct Watcher { - receiver: mpsc::UnboundedReceiver>, +pub struct Watcher { + receiver: mpsc::UnboundedReceiver>, hash: H, } -impl Watcher { +impl Watcher { /// Returns the transaction hash. pub fn hash(&self) -> &H { &self.hash @@ -40,30 +40,30 @@ impl Watcher { /// Pipe the notifications to given sink. /// /// Make sure to drive the future to completion. - pub fn into_stream(self) -> impl Stream> { + pub fn into_stream(self) -> impl Stream> { self.receiver } } /// Sender part of the watcher. Exposed only for testing purposes. #[derive(Debug)] -pub struct Sender { - receivers: Vec>>, - finalized: bool, +pub struct Sender { + receivers: Vec>>, + is_finalized: bool, } -impl Default for Sender { +impl Default for Sender { fn default() -> Self { Sender { receivers: Default::default(), - finalized: false, + is_finalized: false, } } } -impl Sender { +impl Sender { /// Add a new watcher to this sender object. - pub fn new_watcher(&mut self, hash: H) -> Watcher { + pub fn new_watcher(&mut self, hash: H) -> Watcher { let (tx, receiver) = mpsc::unbounded(); self.receivers.push(tx); Watcher { @@ -85,26 +85,42 @@ impl Sender { /// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid. pub fn usurped(&mut self, hash: H) { self.send(TransactionStatus::Usurped(hash)); - self.finalized = true; + self.is_finalized = true; } /// Extrinsic has been included in block with given hash. - pub fn in_block(&mut self, hash: H2) { + pub fn in_block(&mut self, hash: BH) { self.send(TransactionStatus::InBlock(hash)); - self.finalized = true; + } + + /// Extrinsic has been finalized by a finality gadget. + pub fn finalized(&mut self, hash: BH) { + self.send(TransactionStatus::Finalized(hash)); + self.is_finalized = true; + } + + /// The block this extrinsic was included in has been retracted + pub fn finality_timeout(&mut self, hash: BH) { + self.send(TransactionStatus::FinalityTimeout(hash)); + self.is_finalized = true; + } + + /// The block this extrinsic was included in has been retracted + pub fn retracted(&mut self, hash: BH) { + self.send(TransactionStatus::Retracted(hash)); } /// Extrinsic has been marked as invalid by the block builder. pub fn invalid(&mut self) { self.send(TransactionStatus::Invalid); // we mark as finalized as there are no more notifications - self.finalized = true; + self.is_finalized = true; } /// Transaction has been dropped from the pool because of the limit. pub fn dropped(&mut self) { self.send(TransactionStatus::Dropped); - self.finalized = true; + self.is_finalized = true; } /// The extrinsic has been broadcast to the given peers. @@ -114,10 +130,10 @@ impl Sender { /// Returns true if the are no more listeners for this extrinsic or it was finalized. pub fn is_done(&self) -> bool { - self.finalized || self.receivers.is_empty() + self.is_finalized || self.receivers.is_empty() } - fn send(&mut self, status: TransactionStatus) { + fn send(&mut self, status: TransactionStatus) { self.receivers.retain(|sender| sender.unbounded_send(status.clone()).is_ok()) } } diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index bfc13c01fdf53b0992f711fa6c58215aab12ba59..84e06cc33e4f5c83e001dce41d68e882fd2657c7 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -64,7 +64,8 @@ impl FullChainApi where impl sc_transaction_graph::ChainApi for FullChainApi where Block: BlockT, - Client: ProvideRuntimeApi + BlockBody + BlockIdTo + 'static + Send + Sync, + Client: ProvideRuntimeApi + BlockBody + BlockIdTo, + Client: Send + Sync + 'static, Client::Api: TaggedTransactionQueue, sp_api::ApiErrorFor: Send, { diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 7084e1c4a0f5cda89ca7b164402df02a20b499e1..a69323553fa8f53c33021e482ac2eedfd79a881c 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -21,6 +21,7 @@ mod api; pub mod error; +mod revalidation; #[cfg(any(feature = "test-helpers", test))] pub mod testing; @@ -28,19 +29,19 @@ pub mod testing; pub use sc_transaction_graph as txpool; pub use crate::api::{FullChainApi, LightChainApi}; -use std::{collections::HashMap, sync::Arc, pin::Pin, time::Instant}; +use std::{collections::HashMap, sync::Arc, pin::Pin}; use futures::{Future, FutureExt, future::ready}; use parking_lot::Mutex; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, NumberFor, SimpleArithmetic, Extrinsic}, + traits::{Block as BlockT, NumberFor, AtLeast32Bit, Extrinsic}, }; use sp_transaction_pool::{ - TransactionPool, PoolStatus, ImportNotificationStream, - TxHash, TransactionFor, TransactionStatusStreamFor, BlockHash, - MaintainedTransactionPool, PoolFuture, + TransactionPool, PoolStatus, ImportNotificationStream, TxHash, TransactionFor, + TransactionStatusStreamFor, MaintainedTransactionPool, PoolFuture, ChainEvent, }; +use wasm_timer::Instant; /// Basic implementation of transaction pool that can be customized by providing PoolApi. pub struct BasicPool @@ -51,8 +52,10 @@ pub struct BasicPool pool: Arc>, api: Arc, revalidation_strategy: Arc>>>, + revalidation_queue: Arc>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for BasicPool where PoolApi: sc_transaction_graph::ChainApi, @@ -85,13 +88,16 @@ pub enum RevalidationType { impl BasicPool where Block: BlockT, - PoolApi: sc_transaction_graph::ChainApi, + PoolApi: sc_transaction_graph::ChainApi + 'static, { /// Create new basic transaction pool with provided api. + /// + /// It will also optionally return background task that might be started by the + /// caller. pub fn new( options: sc_transaction_graph::Options, pool_api: Arc, - ) -> Self { + ) -> (Self, Option + Send>>>) { Self::with_revalidation_type(options, pool_api, RevalidationType::Full) } @@ -101,19 +107,29 @@ impl BasicPool options: sc_transaction_graph::Options, pool_api: Arc, revalidation_type: RevalidationType, - ) -> Self { - let cloned_api = pool_api.clone(); - BasicPool { - api: cloned_api, - pool: Arc::new(sc_transaction_graph::Pool::new(options, pool_api)), - revalidation_strategy: Arc::new(Mutex::new( - match revalidation_type { - RevalidationType::Light => RevalidationStrategy::Light(RevalidationStatus::NotScheduled), - RevalidationType::Full => RevalidationStrategy::Always, - } - )), - } - + ) -> (Self, Option + Send>>>) { + let pool = Arc::new(sc_transaction_graph::Pool::new(options, pool_api.clone())); + let (revalidation_queue, background_task) = match revalidation_type { + RevalidationType::Light => (revalidation::RevalidationQueue::new(pool_api.clone(), pool.clone()), None), + RevalidationType::Full => { + let (queue, background) = revalidation::RevalidationQueue::new_background(pool_api.clone(), pool.clone()); + (queue, Some(background)) + }, + }; + ( + BasicPool { + api: pool_api, + pool, + revalidation_queue: Arc::new(revalidation_queue), + revalidation_strategy: Arc::new(Mutex::new( + match revalidation_type { + RevalidationType::Light => RevalidationStrategy::Light(RevalidationStatus::NotScheduled), + RevalidationType::Full => RevalidationStrategy::Always, + } + )), + }, + background_task, + ) } /// Gets shared reference to the underlying pool. @@ -172,19 +188,19 @@ impl TransactionPool for BasicPool } fn remove_invalid(&self, hashes: &[TxHash]) -> Vec> { - self.pool.remove_invalid(hashes) + self.pool.validated_pool().remove_invalid(hashes) } fn status(&self) -> PoolStatus { - self.pool.status() + self.pool.validated_pool().status() } fn ready(&self) -> Box>> { - Box::new(self.pool.ready()) + Box::new(self.pool.validated_pool().ready()) } fn import_notification_stream(&self) -> ImportNotificationStream> { - self.pool.import_notification_stream() + self.pool.validated_pool().import_notification_stream() } fn hash_of(&self, xt: &TransactionFor) -> TxHash { @@ -192,11 +208,11 @@ impl TransactionPool for BasicPool } fn on_broadcasted(&self, propagations: HashMap, Vec>) { - self.pool.on_broadcasted(propagations) + self.pool.validated_pool().on_broadcasted(propagations) } fn ready_transaction(&self, hash: &TxHash) -> Option> { - self.pool.ready_transaction(hash) + self.pool.validated_pool().ready_by_hash(hash) } } @@ -205,23 +221,22 @@ enum RevalidationStatus { /// The revalidation has never been completed. NotScheduled, /// The revalidation is scheduled. - Scheduled(Option, Option), + Scheduled(Option, Option), /// The revalidation is in progress. InProgress, } enum RevalidationStrategy { Always, - Light(RevalidationStatus) + Light(RevalidationStatus), } struct RevalidationAction { revalidate: bool, resubmit: bool, - revalidate_amount: Option, } -impl RevalidationStrategy { +impl RevalidationStrategy { pub fn clear(&mut self) { if let Self::Light(status) = self { status.clear() @@ -239,21 +254,19 @@ impl RevalidationStrategy { revalidate: status.next_required( block, revalidate_time_period, - revalidate_block_period + revalidate_block_period, ), resubmit: false, - revalidate_amount: None, }, Self::Always => RevalidationAction { revalidate: true, resubmit: true, - revalidate_amount: Some(16), } } } } -impl RevalidationStatus { +impl RevalidationStatus { /// Called when revalidation is completed. pub fn clear(&mut self) { *self = Self::NotScheduled; @@ -273,7 +286,7 @@ impl RevalidationStatus { revalidate_block_period.map(|period| block + period), ); false - }, + } Self::Scheduled(revalidate_at_time, revalidate_at_block) => { let is_required = revalidate_at_time.map(|at| Instant::now() >= at).unwrap_or(false) || revalidate_at_block.map(|at| block >= at).unwrap_or(false); @@ -281,87 +294,105 @@ impl RevalidationStatus { *self = Self::InProgress; } is_required - }, + } Self::InProgress => false, } } } impl MaintainedTransactionPool for BasicPool -where - Block: BlockT, - PoolApi: 'static + sc_transaction_graph::ChainApi, + where + Block: BlockT, + PoolApi: 'static + sc_transaction_graph::ChainApi, { - fn maintain(&self, id: &BlockId, retracted: &[BlockHash]) - -> Pin + Send>> - { - let id = id.clone(); - let pool = self.pool.clone(); - let api = self.api.clone(); - - let block_number = match api.block_id_to_number(&id) { - Ok(Some(number)) => number, - _ => { - log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id); - return Box::pin(ready(())); - } - }; - - let next_action = self.revalidation_strategy.lock().next( - block_number, - Some(std::time::Duration::from_secs(60)), - Some(20.into()), - ); - let revalidation_strategy = self.revalidation_strategy.clone(); - let retracted = retracted.to_vec(); - - async move { - // We don't query block if we won't prune anything - if !pool.status().is_empty() { - let hashes = api.block_body(&id).await - .unwrap_or_else(|e| { - log::warn!("Prune known transactions: error request {:?}!", e); - None - }) - .unwrap_or_default() - .into_iter() - .map(|tx| pool.hash_of(&tx)) - .collect::>(); - - if let Err(e) = pool.prune_known(&id, &hashes) { - log::error!("Cannot prune known in the pool {:?}!", e); - } - } - - if next_action.resubmit { - let mut resubmit_transactions = Vec::new(); - - for retracted_hash in retracted { - let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await - .unwrap_or_else(|e| { - log::warn!("Failed to fetch block body {:?}!", e); - None - }) - .unwrap_or_default() - .into_iter() - .filter(|tx| tx.is_signed().unwrap_or(true)); - - resubmit_transactions.extend(block_transactions); - } - if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await { - log::debug!(target: "txpool", - "[{:?}] Error re-submitting transactions: {:?}", id, e - ) - } + fn maintain(&self, event: ChainEvent) -> Pin + Send>> { + match event { + ChainEvent::NewBlock { id, retracted, .. } => { + let id = id.clone(); + let pool = self.pool.clone(); + let api = self.api.clone(); + + let block_number = match api.block_id_to_number(&id) { + Ok(Some(number)) => number, + _ => { + log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id); + return Box::pin(ready(())); + } + }; + + let next_action = self.revalidation_strategy.lock().next( + block_number, + Some(std::time::Duration::from_secs(60)), + Some(20.into()), + ); + let revalidation_strategy = self.revalidation_strategy.clone(); + let retracted = retracted.clone(); + let revalidation_queue = self.revalidation_queue.clone(); + + async move { + // We don't query block if we won't prune anything + if !pool.validated_pool().status().is_empty() { + let hashes = api.block_body(&id).await + .unwrap_or_else(|e| { + log::warn!("Prune known transactions: error request {:?}!", e); + None + }) + .unwrap_or_default() + .into_iter() + .map(|tx| pool.hash_of(&tx)) + .collect::>(); + + if let Err(e) = pool.prune_known(&id, &hashes) { + log::error!("Cannot prune known in the pool {:?}!", e); + } + } + + if next_action.resubmit { + let mut resubmit_transactions = Vec::new(); + + for retracted_hash in retracted { + // notify txs awaiting finality that it has been retracted + pool.validated_pool().on_block_retracted(retracted_hash.clone()); + + let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await + .unwrap_or_else(|e| { + log::warn!("Failed to fetch block body {:?}!", e); + None + }) + .unwrap_or_default() + .into_iter() + .filter(|tx| tx.is_signed().unwrap_or(true)); + + resubmit_transactions.extend(block_transactions); + } + if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await { + log::debug!( + target: "txpool", + "[{:?}] Error re-submitting transactions: {:?}", id, e + ) + } + } + + if next_action.revalidate { + let hashes = pool.validated_pool().ready().map(|tx| tx.hash.clone()).collect(); + revalidation_queue.revalidate_later(block_number, hashes).await; + } + + revalidation_strategy.lock().clear(); + }.boxed() } - - if next_action.revalidate { - if let Err(e) = pool.revalidate_ready(&id, next_action.revalidate_amount).await { - log::warn!("Revalidate ready failed {:?}", e); - } + ChainEvent::Finalized { hash } => { + let pool = self.pool.clone(); + async move { + if let Err(e) = pool.validated_pool().on_block_finalized(hash).await { + log::warn!( + target: "txpool", + "Error [{}] occurred while attempting to notify watchers of finalization {}", + e, hash + ) + } + }.boxed() } - - revalidation_strategy.lock().clear(); - }.boxed() + } } } diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs new file mode 100644 index 0000000000000000000000000000000000000000..dbf8a2935427113916c78ceffcb8bde742da78db --- /dev/null +++ b/client/transaction-pool/src/revalidation.rs @@ -0,0 +1,313 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Pool periodic revalidation. + +use std::{sync::Arc, pin::Pin, collections::{HashMap, HashSet, BTreeMap}}; + +use sc_transaction_graph::{ChainApi, Pool, ExHash, NumberFor, ValidatedTransaction}; +use sp_runtime::traits::{Zero, SaturatedConversion}; +use sp_runtime::generic::BlockId; +use sp_runtime::transaction_validity::TransactionValidityError; + +use futures::{prelude::*, channel::mpsc, stream::unfold}; +use std::time::Duration; +use futures_timer::Delay; + +#[cfg(not(test))] +const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(200); +#[cfg(test)] +pub const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(5); + +const BACKGROUND_REVALIDATION_BATCH_SIZE: usize = 20; + +/// Payload from queue to worker. +struct WorkerPayload { + at: NumberFor, + transactions: Vec>, +} + +/// Async revalidation worker. +/// +/// Implements future and can be spawned in place or in background. +struct RevalidationWorker { + api: Arc, + pool: Arc>, + best_block: NumberFor, + block_ordered: BTreeMap, HashSet>>, + members: HashMap, NumberFor>, +} + +impl Unpin for RevalidationWorker {} + +fn interval(duration: Duration) -> impl Stream + Unpin { + unfold((), move |_| { + Delay::new(duration).map(|_| Some(((), ()))) + }).map(drop) +} + +/// Revalidate batch of transaction. +/// +/// Each transaction is validated against chain, and invalid are +/// removed from the `pool`, while valid are resubmitted. +async fn batch_revalidate( + pool: Arc>, + api: Arc, + at: NumberFor, + batch: impl IntoIterator>, +) { + let mut invalid_hashes = Vec::new(); + let mut revalidated = HashMap::new(); + + for ext_hash in batch { + let ext = match pool.validated_pool().ready_by_hash(&ext_hash) { + Some(ext) => ext, + None => continue, + }; + + match api.validate_transaction(&BlockId::Number(at), ext.data.clone()).await { + Ok(Err(TransactionValidityError::Invalid(err))) => { + log::debug!(target: "txpool", "[{:?}]: Revalidation: invalid {:?}", ext_hash, err); + invalid_hashes.push(ext_hash); + }, + Ok(Err(TransactionValidityError::Unknown(err))) => { + // skipping unknown, they might be pushed by valid or invalid transaction + // when latter resubmitted. + log::trace!(target: "txpool", "[{:?}]: Unknown during revalidation: {:?}", ext_hash, err); + }, + Ok(Ok(validity)) => { + revalidated.insert( + ext_hash.clone(), + ValidatedTransaction::valid_at( + at.saturated_into::(), + ext_hash, + ext.data.clone(), + api.hash_and_length(&ext.data).1, + validity, + ) + ); + }, + Err(validation_err) => { + log::debug!( + target: "txpool", + "[{:?}]: Error during revalidation: {:?}. Removing.", + ext_hash, + validation_err + ); + invalid_hashes.push(ext_hash); + } + } + } + + pool.validated_pool().remove_invalid(&invalid_hashes); + pool.resubmit(revalidated); +} + +impl RevalidationWorker { + fn new( + api: Arc, + pool: Arc>, + ) -> Self { + Self { + api, + pool, + block_ordered: Default::default(), + members: Default::default(), + best_block: Zero::zero(), + } + } + + fn prepare_batch(&mut self) -> Vec> { + let mut queued_exts = Vec::new(); + let mut left = BACKGROUND_REVALIDATION_BATCH_SIZE; + + // Take maximum of count transaction by order + // which they got into the pool + while left > 0 { + let first_block = match self.block_ordered.keys().next().cloned() { + Some(bn) => bn, + None => break, + }; + let mut block_drained = false; + if let Some(extrinsics) = self.block_ordered.get_mut(&first_block) { + let to_queue = extrinsics.iter().take(left).cloned().collect::>(); + if to_queue.len() == extrinsics.len() { + block_drained = true; + } else { + for xt in &to_queue { + extrinsics.remove(xt); + } + } + left -= to_queue.len(); + queued_exts.extend(to_queue); + } + + if block_drained { + self.block_ordered.remove(&first_block); + } + } + + queued_exts + } + + fn push(&mut self, worker_payload: WorkerPayload) { + // we don't add something that already scheduled for revalidation + let transactions = worker_payload.transactions; + let block_number = worker_payload.at; + + for ext_hash in transactions { + // we don't add something that already scheduled for revalidation + if self.members.contains_key(&ext_hash) { continue; } + + self.block_ordered.entry(block_number) + .and_modify(|value| { value.insert(ext_hash.clone()); }) + .or_insert_with(|| { + let mut bt = HashSet::new(); + bt.insert(ext_hash.clone()); + bt + }); + self.members.insert(ext_hash.clone(), block_number); + } + } + + /// Background worker main loop. + /// + /// It does two things: periodically tries to process some transactions + /// from the queue and also accepts messages to enqueue some more + /// transactions from the pool. + pub async fn run(mut self, from_queue: mpsc::UnboundedReceiver>) { + let interval = interval(BACKGROUND_REVALIDATION_INTERVAL).fuse(); + let from_queue = from_queue.fuse(); + futures::pin_mut!(interval, from_queue); + let this = &mut self; + + loop { + futures::select! { + _ = interval.next() => { + let next_batch = this.prepare_batch(); + batch_revalidate(this.pool.clone(), this.api.clone(), this.best_block, next_batch).await; + }, + workload = from_queue.next() => { + match workload { + Some(worker_payload) => { + this.best_block = worker_payload.at; + this.push(worker_payload); + continue; + }, + // R.I.P. worker! + None => break, + } + } + } + } + } +} + + +/// Revalidation queue. +/// +/// Can be configured background (`new_background`) +/// or immediate (just `new`). +pub struct RevalidationQueue { + pool: Arc>, + api: Arc, + background: Option>>, +} + +impl RevalidationQueue +where + Api: 'static, +{ + /// New revalidation queue without background worker. + pub fn new(api: Arc, pool: Arc>) -> Self { + Self { + api, + pool, + background: None, + } + } + + /// New revalidation queue with background worker. + pub fn new_background(api: Arc, pool: Arc>) -> + (Self, Pin + Send>>) + { + let (to_worker, from_queue) = mpsc::unbounded(); + + let worker = RevalidationWorker::new(api.clone(), pool.clone()); + + let queue = + Self { + api, + pool, + background: Some(to_worker), + }; + + (queue, worker.run(from_queue).boxed()) + } + + /// Queue some transaction for later revalidation. + /// + /// If queue configured with background worker, this will return immediately. + /// If queue configured without background worker, this will resolve after + /// revalidation is actually done. + pub async fn revalidate_later(&self, at: NumberFor, transactions: Vec>) { + if let Some(ref to_worker) = self.background { + if let Err(e) = to_worker.unbounded_send(WorkerPayload { at, transactions }) { + log::warn!(target: "txpool", "Failed to update background worker: {:?}", e); + } + return; + } else { + let pool = self.pool.clone(); + let api = self.api.clone(); + batch_revalidate(pool, api, at, transactions).await + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use sc_transaction_graph::Pool; + use substrate_test_runtime_transaction_pool::{TestApi, uxt}; + use futures::executor::block_on; + use substrate_test_runtime_client::{ + AccountKeyring::*, + }; + + fn setup() -> (Arc, Pool) { + let test_api = Arc::new(TestApi::empty()); + let pool = Pool::new(Default::default(), test_api.clone()); + (test_api, pool) + } + + #[test] + fn smoky() { + let (api, pool) = setup(); + let pool = Arc::new(pool); + let queue = Arc::new(RevalidationQueue::new(api.clone(), pool.clone())); + + let uxt = uxt(Alice, 0); + let uxt_hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).expect("Should be valid"); + + block_on(queue.revalidate_later(0, vec![uxt_hash])); + + // revalidated in sync offload 2nd time + assert_eq!(api.validation_requests().len(), 2); + // number of ready + assert_eq!(pool.validated_pool().status().ready, 1); + } +} \ No newline at end of file diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index fed02067b184aa0321fe84f7c3c14ee4b772e4bd..6984877eef4aa1b2a769580a6e102bf6289240e6 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -15,24 +15,40 @@ // along with Substrate. If not, see . use crate::*; -use sc_transaction_graph::Pool; +use sp_transaction_pool::TransactionStatus; use futures::executor::block_on; +use txpool::{self, Pool}; use sp_runtime::{ generic::BlockId, transaction_validity::ValidTransaction, }; use substrate_test_runtime_client::{ - runtime::{Block, Hash, Index}, + runtime::{Block, Hash, Index, Header, Extrinsic}, AccountKeyring::*, }; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; +use crate::revalidation::BACKGROUND_REVALIDATION_INTERVAL; fn pool() -> Pool { Pool::new(Default::default(), TestApi::with_alice_nonce(209).into()) } -fn maintained_pool() -> BasicPool { - BasicPool::new(Default::default(), std::sync::Arc::new(TestApi::with_alice_nonce(209))) +fn maintained_pool() -> (BasicPool, futures::executor::ThreadPool) { + let (pool, background_task) = BasicPool::new(Default::default(), std::sync::Arc::new(TestApi::with_alice_nonce(209))); + + let thread_pool = futures::executor::ThreadPool::new().unwrap(); + thread_pool.spawn_ok(background_task.expect("basic pool have background task")); + (pool, thread_pool) +} + +fn header(number: u64) -> Header { + Header { + number, + digest: Default::default(), + extrinsics_root: Default::default(), + parent_hash: Default::default(), + state_root: Default::default(), + } } #[test] @@ -40,7 +56,7 @@ fn submission_should_work() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209]); } @@ -50,7 +66,7 @@ fn multiple_submission_should_work() { block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); } @@ -59,7 +75,7 @@ fn early_nonce_should_be_culled() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 208))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); } @@ -68,11 +84,11 @@ fn late_nonce_should_be_queued() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); } @@ -82,7 +98,7 @@ fn prune_tags_should_work() { let hash209 = block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); block_on( @@ -93,7 +109,7 @@ fn prune_tags_should_work() { ) ).expect("Prune tags"); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![210]); } @@ -102,11 +118,11 @@ fn should_ban_invalid_transactions() { let pool = pool(); let uxt = uxt(Alice, 209); let hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap(); - pool.remove_invalid(&[hash]); + pool.validated_pool().remove_invalid(&[hash]); block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap_err(); // when - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); // then @@ -122,43 +138,62 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { let pool = Pool::new(Default::default(), api.clone()); let xt = uxt(Alice, 209); block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // remove the transaction that just got imported. api.increment_nonce(Alice.into()); block_on(pool.prune_tags(&BlockId::number(1), vec![vec![209]], vec![])).expect("1. Pruned"); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // it's re-imported to future - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); // so now let's insert another transaction that also provides the 155 api.increment_nonce(Alice.into()); let xt = uxt(Alice, 211); block_on(pool.submit_one(&BlockId::number(2), xt.clone())).expect("2. Imported"); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 1); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 1); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![211]); // prune it and make sure the pool is empty api.increment_nonce(Alice.into()); block_on(pool.prune_tags(&BlockId::number(3), vec![vec![155]], vec![])).expect("2. Pruned"); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 2); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 2); +} + +fn block_event(id: u64) -> ChainEvent { + ChainEvent::NewBlock { + id: BlockId::number(id), + is_new_best: true, + retracted: vec![], + header: header(id), + } +} + +fn block_event_with_retracted(id: u64, retracted: Vec) -> ChainEvent { + ChainEvent::NewBlock { + id: BlockId::number(id), + is_new_best: true, + retracted: retracted, + header: header(id), + } } + #[test] fn should_prune_old_during_maintenance() { let xt = uxt(Alice, 209); - let pool = maintained_pool(); + let (pool, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); pool.api.push_block(1, vec![xt.clone()]); - block_on(pool.maintain(&BlockId::number(1), &[])); + block_on(pool.maintain(block_event(1))); assert_eq!(pool.status().ready, 0); } @@ -167,7 +202,7 @@ fn should_revalidate_during_maintenance() { let xt1 = uxt(Alice, 209); let xt2 = uxt(Alice, 210); - let pool = maintained_pool(); + let (pool, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), xt1.clone())).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), xt2.clone())).expect("2. Imported"); assert_eq!(pool.status().ready, 2); @@ -175,18 +210,23 @@ fn should_revalidate_during_maintenance() { pool.api.push_block(1, vec![xt1.clone()]); - block_on(pool.maintain(&BlockId::number(1), &[])); + block_on(pool.maintain(block_event(1))); + + // maintaince is in background + block_on(futures_timer::Delay::new(BACKGROUND_REVALIDATION_INTERVAL*2)); + + block_on(pool.maintain(block_event(1))); assert_eq!(pool.status().ready, 1); // test that pool revalidated transaction that left ready and not included in the block assert_eq!(pool.api.validation_requests().len(), 3); } #[test] -fn should_resubmit_from_retracted_during_maintaince() { +fn should_resubmit_from_retracted_during_maintenance() { let xt = uxt(Alice, 209); let retracted_hash = Hash::random(); - let pool = maintained_pool(); + let (pool, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); @@ -194,7 +234,9 @@ fn should_resubmit_from_retracted_during_maintaince() { pool.api.push_block(1, vec![]); pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); - block_on(pool.maintain(&BlockId::number(1), &[retracted_hash])); + let event = block_event_with_retracted(1, vec![retracted_hash]); + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 1); } @@ -203,7 +245,7 @@ fn should_not_retain_invalid_hashes_from_retracted() { let xt = uxt(Alice, 209); let retracted_hash = Hash::random(); - let pool = maintained_pool(); + let (pool, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); @@ -212,17 +254,293 @@ fn should_not_retain_invalid_hashes_from_retracted() { pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); pool.api.add_invalid(&xt); - block_on(pool.maintain(&BlockId::number(1), &[retracted_hash])); + let event = block_event_with_retracted(1, vec![retracted_hash]); + + block_on(pool.maintain(event)); + + // maintenance is in background + block_on(futures_timer::Delay::new(BACKGROUND_REVALIDATION_INTERVAL*2)); + + let event = block_event_with_retracted(1, vec![retracted_hash]); + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); } +#[test] +fn should_push_watchers_during_maintaince() { + fn alice_uxt(nonce: u64) -> Extrinsic { + uxt(Alice, 209 + nonce) + } + + // given + let (pool, _guard) = maintained_pool(); + + let tx0 = alice_uxt(0); + let watcher0 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx0.clone())).unwrap(); + let tx1 = alice_uxt(1); + let watcher1 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx1.clone())).unwrap(); + let tx2 = alice_uxt(2); + let watcher2 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx2.clone())).unwrap(); + let tx3 = alice_uxt(3); + let watcher3 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx3.clone())).unwrap(); + let tx4 = alice_uxt(4); + let watcher4 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx4.clone())).unwrap(); + assert_eq!(pool.status().ready, 5); + + // when + pool.api.add_invalid(&tx3); + pool.api.add_invalid(&tx4); + block_on(pool.maintain(block_event(0))); + + // revalidation is in background + block_on(futures_timer::Delay::new(BACKGROUND_REVALIDATION_INTERVAL*2)); + + // then + // hash3 is now invalid + // hash4 is now invalid + + assert_eq!(pool.status().ready, 3); + assert_eq!( + futures::executor::block_on_stream(watcher3).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::Invalid], + ); + assert_eq!( + futures::executor::block_on_stream(watcher4).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::Invalid], + ); + + // when + let header_hash = pool.api.push_block(1, vec![tx0, tx1, tx2]).hash(); + block_on(pool.maintain(block_event(1))); + + let event = ChainEvent::Finalized { hash: header_hash.clone() }; + block_on(pool.maintain(event)); + + // then + // events for hash0 are: Ready, InBlock + // events for hash1 are: Ready, InBlock + // events for hash2 are: Ready, InBlock + assert_eq!( + futures::executor::block_on_stream(watcher0).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + ); + assert_eq!( + futures::executor::block_on_stream(watcher1).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + ); + assert_eq!( + futures::executor::block_on_stream(watcher2).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + ); +} + #[test] fn can_track_heap_size() { - let pool = maintained_pool(); + let (pool, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 211))).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 212))).expect("1. Imported"); assert!(parity_util_mem::malloc_size(&pool) > 3000); -} \ No newline at end of file +} + +#[test] +fn finalization() { + let xt = uxt(Alice, 209); + let api = TestApi::with_alice_nonce(209); + api.push_block(1, vec![]); + let (pool, _background) = BasicPool::new(Default::default(), api.into()); + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + pool.api.push_block(2, vec![xt.clone()]); + + let header = pool.api.chain().read().header_by_number.get(&2).cloned().unwrap(); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + block_on(pool.maintain(event)); + + let event = ChainEvent::Finalized { hash: header.hash() }; + block_on(pool.maintain(event)); + + let mut stream = futures::executor::block_on_stream(watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(header.hash()))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(header.hash()))); + assert_eq!(stream.next(), None); +} + +#[test] +fn fork_aware_finalization() { + let api = TestApi::empty(); + // starting block A1 (last finalized.) + api.push_block(1, vec![]); + + let (pool, _background) = BasicPool::new(Default::default(), api.into()); + let mut canon_watchers = vec![]; + + let from_alice = uxt(Alice, 1); + let from_dave = uxt(Dave, 2); + let from_bob = uxt(Bob, 1); + let from_charlie = uxt(Charlie, 1); + pool.api.increment_nonce(Alice.into()); + pool.api.increment_nonce(Dave.into()); + pool.api.increment_nonce(Charlie.into()); + pool.api.increment_nonce(Bob.into()); + + let from_dave_watcher; + let from_bob_watcher; + let b1; + let d1; + let c2; + let d2; + + + // block B1 + { + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_alice.clone())).expect("1. Imported"); + let header = pool.api.push_block(2, vec![from_alice.clone()]); + canon_watchers.push((watcher, header.hash())); + assert_eq!(pool.status().ready, 1); + + let event = ChainEvent::NewBlock { + id: BlockId::Number(2), + is_new_best: true, + header: header.clone(), + retracted: vec![], + }; + b1 = header.hash(); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + let event = ChainEvent::Finalized { hash: b1 }; + block_on(pool.maintain(event)); + } + + // block C2 + { + let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]); + from_dave_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_dave.clone())) + .expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + c2 = header.hash(); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + } + + // block D2 + { + from_bob_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_bob.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]); + + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + d2 = header.hash(); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + } + + // block C1 + { + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_charlie.clone())).expect("1.Imported"); + assert_eq!(pool.status().ready, 1); + let header = pool.api.push_block(3, vec![from_charlie.clone()]); + + canon_watchers.push((watcher, header.hash())); + let event = ChainEvent::NewBlock { + id: BlockId::Number(3), + is_new_best: true, + header: header.clone(), + retracted: vec![c2, d2], + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); + let event = ChainEvent::Finalized { hash: header.hash() }; + block_on(pool.maintain(event)); + } + + // block D1 + { + let xt = uxt(Eve, 0); + let w = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 3); + let header = pool.api.push_block(4, vec![xt.clone()]); + canon_watchers.push((w, header.hash())); + + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + d1 = header.hash(); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); + let event = ChainEvent::Finalized { hash: d1 }; + block_on(pool.maintain(event)); + } + + let e1; + + // block e1 + { + let header = pool.api.push_block(5, vec![from_dave, from_bob]); + e1 = header.hash(); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + block_on(pool.maintain(ChainEvent::Finalized { hash: e1 })); + } + + + for (canon_watcher, h) in canon_watchers { + let mut stream = futures::executor::block_on_stream(canon_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(h.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(h))); + assert_eq!(stream.next(), None); + } + + + { + let mut stream= futures::executor::block_on_stream(from_dave_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(c2.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Retracted(c2))); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone()))); + assert_eq!(stream.next(), None); + } + + { + let mut stream= futures::executor::block_on_stream(from_bob_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(d2.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Retracted(d2))); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone()))); + assert_eq!(stream.next(), None); + } + +} diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index 9304f278534bbe4faa0bc7523c7a27bb99781ce5..701df299a35453ce71a037150d46d329847d5f80 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -75,3 +75,6 @@ # Authority discovery /client/authority-discovery/ @mxinden /frame/authority-discovery/ @mxinden + +# Prometheus endpoint +/utils/prometheus/ @mxinden diff --git a/docs/CODE_OF_CONDUCT.adoc b/docs/CODE_OF_CONDUCT.adoc index 7cb0210e8efac2575e23bd46a000af59bb36cb3e..0f7de7c7efee14bc03258e0aa4f5e7771a912067 100644 --- a/docs/CODE_OF_CONDUCT.adoc +++ b/docs/CODE_OF_CONDUCT.adoc @@ -22,7 +22,7 @@ Examples of unacceptable behavior by participants include: * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting -=== Facilitation, Not Strongarming +=== Facilitation, Not Strong Arming We recognize that this software is merely a tool for users to create and maintain their blockchain of preference. We see that blockchains are naturally community platforms with users being the ultimate decision makers. We assert that good software will maximize user agency by facilitate user-expression on the network. As such: diff --git a/docs/CONTRIBUTING.adoc b/docs/CONTRIBUTING.adoc index 817e1d7489f3968c2a851e78b1728949776b544d..cdd9809fff864cdb1b26321d76567ac0152233e5 100644 --- a/docs/CONTRIBUTING.adoc +++ b/docs/CONTRIBUTING.adoc @@ -27,7 +27,7 @@ Merging pull requests once CI is successful: . Once a PR is ready for review please add the https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview[`pleasereview`] label. Generally PRs should sit with this label for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. . If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-looksgood[`A8-looksgood`], it is ready to merge. -. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`breaksapi`], when it changes the SRML or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`breaksconsensus`] +. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`breaksconsensus`] . No PR should be merged until all reviews' comments are addressed. *Reviewing pull requests*: @@ -52,7 +52,7 @@ If your PR changes the external APIs or interfaces used by Polkadot, **a corresp To update a corresponding Polkadot PR: -0. Pull lastet Polkadot master (or clone it, if you haven't yet). +0. Pull latest Polkadot master (or clone it, if you haven't yet). 1. Replace `polkadot-master` in all `Cargo.toml` with the name of the PR-branch - e.g. by running `find . -name "Cargo.toml" -exec sed -i "s/polkadot-master/PR_BRANCH/g" {}` (and to your repo, if the branch is not on mainline); Commit this change. 2. Make the changes required to pass the build again. 3. Submit all this as a PR against the Polkadot Repo, link that new PR in the existing substrate PR for reference diff --git a/docs/README.adoc b/docs/README.adoc index d8c582296cabbfa396a7e1761a07f0555d3f1fd1..8d762fee05f8789f164b0c5a263e334c28668176 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -24,13 +24,13 @@ Substrate is designed for use in one of three ways: **1. Trivial**: By running the Substrate binary `substrate` and configuring it with a genesis block that includes the current demonstration runtime. In this case, you just build Substrate, configure a JSON file, and launch your own blockchain. This affords you the least amount of customizability, primarily allowing you to change the genesis parameters of the various included runtime modules such as balances, staking, block-period, fees, and governance. -**2. Modular**: By hacking together pallets built with Substrate FRAME into a new runtime and possibly altering or reconfiguring the Substrate client's block authoring logic. This affords you a very large amount of freedom over your blockchain's logic, letting you change datatypes, add or remove modules, and crucially, add your own modules. Much can be changed without touching the block authoring logic (since it is generic). If this is the case, then the existing Substrate binary can be used for block authoring and syncing. If the block authoring logic needs to be tweaked, then a new, altered block authoring binary must be built as a separate project and used by validators. This is how the Polkadot relay chain is built and should suffice for almost all circumstances in the near to mid-term. +**2. Modular**: By hacking together pallets built with Substrate FRAME into a new runtime and possibly altering or reconfiguring the Substrate client's block authoring logic. This affords you a very large amount of freedom over your blockchain's logic, letting you change data types, add or remove modules, and crucially, add your own modules. Much can be changed without touching the block authoring logic (since it is generic). If this is the case, then the existing Substrate binary can be used for block authoring and syncing. If the block authoring logic needs to be tweaked, then a new, altered block authoring binary must be built as a separate project and used by validators. This is how the Polkadot relay chain is built and should suffice for almost all circumstances in the near to mid-term. -**3. Generic**: The entire SRML can be ignored and the entire runtime designed and implemented from scratch. If desired, this can be done in a language other than Rust, provided it can target WebAssembly. If the runtime can be made compatible with the existing client's block authoring logic, then you can simply construct a new genesis block from your Wasm blob and launch your chain with the existing Rust-based Substrate client. If not, then you'll need to alter the client's block authoring logic accordingly. This is probably a useless option for most projects right now, but provides complete flexibility allowing for a long-term, far-reaching upgrade path for the Substrate paradigm. +**3. Generic**: The entire FRAME can be ignored and the entire runtime designed and implemented from scratch. If desired, this can be done in a language other than Rust, provided it can target WebAssembly. If the runtime can be made compatible with the existing client's block authoring logic, then you can simply construct a new genesis block from your Wasm blob and launch your chain with the existing Rust-based Substrate client. If not, then you'll need to alter the client's block authoring logic accordingly. This is probably a useless option for most projects right now, but provides complete flexibility allowing for a long-term, far-reaching upgrade path for the Substrate paradigm. === The Basics of Substrate -Substrate is a blockchain platform with a completely generic state transition function. That said, it does come with both standards and conventions (particularly regarding the Runtime Module Library) regarding underlying data structures. Roughly speaking, these core datatypes correspond to +trait+s in terms of the actual non-negotiable standard and generic +struct+s in terms of the convention. +Substrate is a blockchain platform with a completely generic state transition function. That said, it does come with both standards and conventions (particularly regarding the Runtime Module Library) regarding underlying data structures. Roughly speaking, these core data types correspond to +trait+s in terms of the actual non-negotiable standard and generic +struct+s in terms of the convention. ``` Header := Parent + ExtrinsicsRoot + StorageRoot + Digest diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b850e5462e2a8368c1e58a417eb9cef229b49537..7240218fa87296309fea68ee4f51471d923292a8 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -36,7 +36,7 @@ Responsible investigation and reporting includes, but isn't limited to, the foll ## Bug Bounty Program -Our Bug Bounty Program allows us to recognise and reward members of the Parity community for helping us find and address significant bugs, in accordance with the terms of the Parity Bug Bounty Program. A detailed description on eligibility, rewards, legal information and terms & conditions for contributors can be found on [our website](https://paritytech.io/bug-bounty.html). +Our Bug Bounty Program allows us to recognize and reward members of the Parity community for helping us find and address significant bugs, in accordance with the terms of the Parity Bug Bounty Program. A detailed description on eligibility, rewards, legal information and terms & conditions for contributors can be found on [our website](https://paritytech.io/bug-bounty.html). diff --git a/docs/Structure.adoc b/docs/Structure.adoc index cb0e4b28cb001fb77e62e41a98775b893c447a8e..c8cd63506a347f3252a8f5d40c9cf3d2dd4db7d8 100644 --- a/docs/Structure.adoc +++ b/docs/Structure.adoc @@ -67,7 +67,7 @@ There are a few crates with the `frame-` prefix. These do not contain domain-spe ** only helpers may be published ** purely testing crates must be `publish = false` -All tests that have to pull (dev)-dependencies out of their subtree and would thus break the dependency rules are considered intergration tests and should be stored in here. Only helper-crates in here shall be published, everything else is expected to be non-publish. +All tests that have to pull (dev)-dependencies out of their subtree and would thus break the dependency rules are considered integration tests and should be stored in here. Only helper-crates in here shall be published, everything else is expected to be non-publish. === Binaries and template diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index cd6b41c4c7a3c5523b0622c968b7f1fd4b9c055b..f9121d9d02d5504db84ebd68300b46d9bf28527a 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -1,24 +1,26 @@ [package] name = "pallet-assets" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } # Needed for various traits. In our case, `OnFinalize`. -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. -frame-support = { version = "2.0.0", default-features = false, path = "../support" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } # `system` module provides us with all sorts of useful stuff and macros depend on it being around. -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-std = { version = "2.0.0", path = "../../primitives/std" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 8a62fa9d8298863c6aab29a6d91e2150d9bff846..042ff89913417ac710a20f348662ea6a2d10eceb 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -133,7 +133,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use frame_support::{Parameter, decl_module, decl_event, decl_storage, decl_error, ensure}; -use sp_runtime::traits::{Member, SimpleArithmetic, Zero, StaticLookup}; +use sp_runtime::traits::{Member, AtLeast32Bit, Zero, StaticLookup}; use frame_system::{self as system, ensure_signed}; use sp_runtime::traits::One; @@ -143,10 +143,10 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// The units in which we record balances. - type Balance: Member + Parameter + SimpleArithmetic + Default + Copy; + type Balance: Member + Parameter + AtLeast32Bit + Default + Copy; /// The arithmetic type of asset identifier. - type AssetId: Parameter + SimpleArithmetic + Default + Copy; + type AssetId: Parameter + AtLeast32Bit + Default + Copy; } decl_module! { @@ -265,9 +265,9 @@ mod tests { pub enum Origin for Test where system = frame_system {} } - // For testing the module, we construct most of a mock runtime. This means + // For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the - // configuration traits of modules we want to use. + // configuration traits of pallets we want to use. #[derive(Clone, Eq, PartialEq)] pub struct Test; parameter_types! { @@ -293,6 +293,9 @@ mod tests { type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Event = (); diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index e982a02b47c8db39d755af9739123c4f909562a0..8e2da7547c8704117b4977df26cfedfd50a0b6a5 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -1,25 +1,27 @@ [package] name = "pallet-aura" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-application-crypto = { version = "2.0.0", default-features = false, path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/application-crypto" } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-inherents = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/inherents" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0", default-features = false, path = "../session" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-io ={ path = "../../primitives/io", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false} -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../primitives/timestamp" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } +pallet-session = { version = "2.0.0-alpha.1", default-features = false, path = "../session" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.1"} +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false, version = "0.8.0-alpha.1"} +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/timestamp" } +pallet-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../timestamp" } [dev-dependencies] diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 81366cf0842d62a587b6b35b75fb5c2a1254f17a..05a161ee49c3d6c9ac7c51028d31f1dd9d193bc8 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -61,6 +61,9 @@ impl frame_system::Trait for Test { type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl pallet_timestamp::Trait for Test { diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index d8c72683cac786067793c4883d167f567dc15c18..7d79bf20f3120fea28316f6749b68cf4d9fea762 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -1,25 +1,27 @@ [package] name = "pallet-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-authority-discovery = { version = "2.0.0", default-features = false, path = "../../primitives/authority-discovery" } -sp-application-crypto = { version = "2.0.0", default-features = false, path = "../../primitives/application-crypto" } +sp-authority-discovery = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/authority-discovery" } +sp-application-crypto = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/application-crypto" } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -pallet-session = { version = "2.0.0", features = ["historical" ], path = "../session", default-features = false } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +pallet-session = { version = "2.0.0-alpha.1", features = ["historical" ], path = "../session", default-features = false } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] -sp-staking = { version = "2.0.0", default-features = false, path = "../../primitives/staking" } +sp-staking = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/staking" } [features] default = ["std"] diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 22ea3d3bbafbbf5a065663716e83b784f49e9e17..8ee4931e488c3740174706480188dfbbdd5b1857 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -157,6 +157,9 @@ mod tests { type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl_outer_origin! { diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index 124d66c1bc33f25cfab28fb3a28dc3f262c182df..f1d4c734a2818f6b8a42779110073888890528ad 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -1,21 +1,23 @@ [package] name = "pallet-authorship" -version = "2.0.0" -description = "Block and Uncle Author tracking for the SRML" +version = "2.0.0-alpha.1" +description = "Block and Uncle Author tracking for the FRAME" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-authorship = { version = "2.0.0", default-features = false, path = "../../primitives/authorship" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-io ={ path = "../../primitives/io", default-features = false } +sp-inherents = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/inherents" } +sp-authorship = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/authorship" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.1"} impl-trait-for-tuples = "0.1.3" [features] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index 216f5c729a9c4be709442245b28128bcbf69e700..d3c1bf752aeb8a197333946e3d4d4f8a751b765f 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Authorship tracking for SRML runtimes. +//! Authorship tracking for FRAME runtimes. //! //! This tracks the current author of the block and recent uncles. @@ -431,6 +431,9 @@ mod tests { type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } parameter_types! { diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index 5e7764862a66b4c58f5b9fcb57e58a6325d97659..88e32d6a074feab4c31e392dc8de3f4074d40a2f 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -1,32 +1,34 @@ [package] name = "pallet-babe" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] hex-literal = "0.2.1" codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../primitives/timestamp" } -pallet-session = { version = "2.0.0", default-features = false, path = "../session" } -sp-consensus-babe = { version = "0.8", default-features = false, path = "../../primitives/consensus/babe" } -sp-io ={ path = "../../primitives/io", default-features = false } +sp-inherents = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../timestamp" } +sp-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/timestamp" } +pallet-session = { version = "2.0.0-alpha.1", default-features = false, path = "../session" } +sp-consensus-babe = { version = "0.8.0-alpha.1", default-features = false, path = "../../primitives/consensus/babe" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.1"} [dev-dependencies] lazy_static = "1.4.0" parking_lot = "0.10.0" -sp-version = { version = "2.0.0", default-features = false, path = "../../primitives/version" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } +sp-version = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/version" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } [features] default = ["std"] diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 1578d5c556c5536b33a0916ec3d50d3a2a65bccf..4dc9304fa8467f7e2bf36dcda500fb8fda37962e 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -23,10 +23,10 @@ pub use pallet_timestamp; use sp_std::{result, prelude::*}; -use frame_support::{decl_storage, decl_module, traits::FindAuthor, traits::Get}; +use frame_support::{decl_storage, decl_module, traits::{FindAuthor, Get, Randomness as RandomnessT}}; use sp_timestamp::OnTimestampSet; -use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill}; -use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, RandomnessBeacon}; +use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill, PerThing}; +use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash}; use sp_staking::{ SessionIndex, offence::{Offence, Kind}, @@ -159,7 +159,7 @@ decl_storage! { } decl_module! { - /// The BABE SRML module + /// The BABE Pallet pub struct Module for enum Call where origin: T::Origin { /// The number of **slots** that an epoch takes. We couple sessions to /// epochs, i.e. we start a new session once the new epoch begins. @@ -191,9 +191,13 @@ decl_module! { } } -impl RandomnessBeacon for Module { - fn random() -> [u8; VRF_OUTPUT_LENGTH] { - Self::randomness() +impl RandomnessT<::Hash> for Module { + fn random(subject: &[u8]) -> T::Hash { + let mut subject = subject.to_vec(); + subject.reserve(VRF_OUTPUT_LENGTH); + subject.extend_from_slice(&Self::randomness()[..]); + + ::Hashing::hash(&subject[..]) } } @@ -244,7 +248,6 @@ impl pallet_session::ShouldEndSession for Module { /// A BABE equivocation offence report. /// /// When a validator released two or more blocks at the same slot. -#[allow(dead_code)] struct BabeEquivocationOffence { /// A babe slot number in which this incident happened. slot: u64, @@ -361,7 +364,7 @@ impl Module { // finds the start slot of the current epoch. only guaranteed to // give correct results after `do_initialize` of the first block // in the chain (as its result is based off of `GenesisSlot`). - fn current_epoch_start() -> SlotNumber { + pub fn current_epoch_start() -> SlotNumber { (EpochIndex::get() * T::EpochDuration::get()) + GenesisSlot::get() } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index e65f305dc4de2b17a4b7be94c07d1ae8e292e4ab..2ec083728e82ceb8488fa9c24efb7223ae5f450e 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -15,17 +15,15 @@ // along with Substrate. If not, see . //! Test utilities -#![allow(dead_code, unused_imports)] use super::{Trait, Module, GenesisConfig}; -use sp_consensus_babe::AuthorityId; use sp_runtime::{ traits::IdentityLookup, Perbill, testing::{Header, UintAuthorityId}, impl_opaque_keys, }; use sp_version::RuntimeVersion; use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; use sp_io; -use sp_core::{H256, Blake2Hasher}; +use sp_core::H256; impl_outer_origin!{ pub enum Origin for Test where system = frame_system {} @@ -66,6 +64,9 @@ impl frame_system::Trait for Test { type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl_opaque_keys! { diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs index 976a264d7ba7bb24f29c556ec92822a728598075..84f8166b10431a9d107cf82d831d4aaa3230dfe1 100644 --- a/frame/babe/src/tests.rs +++ b/frame/babe/src/tests.rs @@ -17,7 +17,7 @@ //! Consensus extension module tests for BABE consensus. use super::*; -use mock::{new_test_ext, Babe, Test}; +use mock::{new_test_ext, Babe, System}; use sp_runtime::{traits::OnFinalize, testing::{Digest, DigestItem}}; use pallet_session::ShouldEndSession; @@ -66,8 +66,6 @@ fn check_module() { }) } -type System = frame_system::Module; - #[test] fn first_block_epoch_zero_start() { new_test_ext(vec![0, 1, 2, 3]).execute_with(|| { diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index 871290b182fdd0ec6e177ac0f1b1cf9557929289..4356b43e6140a9df14520ad2eb9911577e0d629c 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "pallet-balances" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-alpha.1", default-features = false, path = "../benchmarking" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -pallet-transaction-payment = { version = "2.0.0", path = "../transaction-payment" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +pallet-transaction-payment = { version = "2.0.0-alpha.1", path = "../transaction-payment" } [features] default = ["std"] @@ -27,6 +29,7 @@ std = [ "sp-std/std", "sp-io/std", "sp-runtime/std", + "frame-benchmarking/std", "frame-support/std", "frame-system/std", ] diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..371692b650f9c0d1c219cdace0e5f2d0f7d931a9 --- /dev/null +++ b/frame/balances/src/benchmarking.rs @@ -0,0 +1,119 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Balances pallet benchmarking. + +use super::*; + +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::{Bounded, Dispatchable}; + +use crate::Module as Balances; + +const SEED: u32 = 0; +const MAX_EXISTENTIAL_DEPOSIT: u32 = 1000; +const MAX_USER_INDEX: u32 = 1000; + +benchmarks! { + _ { + let e in 2 .. MAX_EXISTENTIAL_DEPOSIT => (); + let u in 1 .. MAX_USER_INDEX => (); + } + + // Benchmark `transfer` extrinsic with the worst possible conditions: + // * Transfer will kill the sender account. + // * Transfer will create the recipient account. + transfer { + let u in ...; + let e in ...; + + let existential_deposit = T::ExistentialDeposit::get(); + let caller = account("caller", u, SEED); + + // Give some multiple of the existential deposit + creation fee + transfer fee + let balance = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&caller, balance); + + // Transfer `e - 1` existential deposits + 1 unit, which guarantees to create one account, and reap this user. + let recipient = account("recipient", u, SEED); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); + let transfer_amount = existential_deposit.saturating_mul((e - 1).into()) + 1.into(); + }: _(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) + + // Benchmark `transfer` with the best possible condition: + // * Both accounts exist and will continue to exist. + transfer_best_case { + let u in ...; + let e in ...; + + let caller = account("caller", u, SEED); + let recipient: T::AccountId = account("recipient", u, SEED); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); + + // Give the sender account max funds for transfer (their account will never reasonably be killed). + let _ = as Currency<_>>::make_free_balance_be(&caller, T::Balance::max_value()); + + // Give the recipient account existential deposit (thus their account already exists). + let existential_deposit = T::ExistentialDeposit::get(); + let _ = as Currency<_>>::make_free_balance_be(&recipient, existential_deposit); + let transfer_amount = existential_deposit.saturating_mul(e.into()); + }: transfer(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) + + // Benchmark `transfer_keep_alive` with the worst possible condition: + // * The recipient account is created. + transfer_keep_alive { + let u in ...; + let e in ...; + + let caller = account("caller", u, SEED); + let recipient = account("recipient", u, SEED); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); + + // Give the sender account max funds, thus a transfer will not kill account. + let _ = as Currency<_>>::make_free_balance_be(&caller, T::Balance::max_value()); + let existential_deposit = T::ExistentialDeposit::get(); + let transfer_amount = existential_deposit.saturating_mul(e.into()); + }: _(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) + + // Benchmark `set_balance` coming from ROOT account. This always creates an account. + set_balance { + let u in ...; + let e in ...; + + let user: T::AccountId = account("user", u, SEED); + let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); + + // Give the user some initial balance. + let existential_deposit = T::ExistentialDeposit::get(); + let balance_amount = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); + }: _(RawOrigin::Root, user_lookup, balance_amount, balance_amount) + + // Benchmark `set_balance` coming from ROOT account. This always kills an account. + set_balance_killing { + let u in ...; + let e in ...; + + let user: T::AccountId = account("user", u, SEED); + let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); + + // Give the user some initial balance. + let existential_deposit = T::ExistentialDeposit::get(); + let balance_amount = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); + }: set_balance(RawOrigin::Root, user_lookup, 0.into(), 0.into()) +} \ No newline at end of file diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 4b3e88eb9f2b1462f748d38d18d3eaa66fe01c67..4dbaf4b8a886d38a38e469f30f68bf5f26f23f58 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -92,7 +92,7 @@ //! //! The following examples show how to use the Balances module in your custom module. //! -//! ### Examples from the SRML +//! ### Examples from the FRAME //! //! The Contract module uses the `Currency` trait to handle gas payment, and its types inherit from `Currency`: //! @@ -149,74 +149,58 @@ #![cfg_attr(not(feature = "std"), no_std)] #[cfg(test)] -mod mock; +mod tests_local; #[cfg(test)] +mod tests_composite; +#[cfg(test)] +#[macro_use] mod tests; -mod migration; +mod benchmarking; use sp_std::prelude::*; -use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr}; +use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; +use sp_io::hashing::twox_64; use codec::{Codec, Encode, Decode}; use frame_support::{ StorageValue, Parameter, decl_event, decl_storage, decl_module, decl_error, ensure, - weights::SimpleDispatchInfo, traits::{ - UpdateBalanceOutcome, Currency, OnReapAccount, OnUnbalanced, TryDrop, + weights::SimpleDispatchInfo, traits::{ + Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap, WithdrawReason, WithdrawReasons, LockIdentifier, LockableCurrency, ExistenceRequirement, - Imbalance, SignedImbalance, ReservableCurrency, Get, ExistenceRequirement::KeepAlive + Imbalance, SignedImbalance, ReservableCurrency, Get, ExistenceRequirement::KeepAlive, + ExistenceRequirement::AllowDeath, IsDeadAccount, BalanceStatus as Status } }; use sp_runtime::{ RuntimeDebug, DispatchResult, DispatchError, traits::{ - Zero, SimpleArithmetic, StaticLookup, Member, CheckedAdd, CheckedSub, + Zero, AtLeast32Bit, StaticLookup, Member, CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Saturating, Bounded, }, }; -use frame_system::{self as system, IsDeadAccount, OnNewAccount, ensure_signed, ensure_root}; -use migration::{get_storage_value, put_storage_value, StorageIterator}; +use frame_system::{self as system, ensure_signed, ensure_root}; +use frame_support::storage::migration::{ + get_storage_value, take_storage_value, put_storage_value, StorageIterator, have_storage_value +}; pub use self::imbalances::{PositiveImbalance, NegativeImbalance}; pub trait Subtrait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + SimpleArithmetic + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; - /// A function that is invoked when the free-balance and the reserved-balance has fallen below - /// the existential deposit and both have been reduced to zero. - /// - /// All resources should be cleaned up all resources associated with the given account. - type OnReapAccount: OnReapAccount; - - /// Handler for when a new account is created. - type OnNewAccount: OnNewAccount; - /// The minimum amount required to keep an account open. type ExistentialDeposit: Get; - /// The fee required to create an account. If you're doing significant stuff with `OnNewAccount` - /// then you'll probably want to make this non-zero. - type CreationFee: Get; + /// The means of storing the balances of an account. + type AccountStore: StoredMap>; } pub trait Trait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + SimpleArithmetic + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; - /// A function that is invoked when the free-balance and the reserved-balance has fallen below - /// the existential deposit and both have been reduced to zero. - /// - /// All resources should be cleaned up all resources associated with the given account. - type OnReapAccount: OnReapAccount; - - /// Handler for when a new account is created. - type OnNewAccount: OnNewAccount; - - /// Handler for the unbalanced reduction when taking fees associated with balance - /// transfer (which may also include account creation). - type TransferPayment: OnUnbalanced>; - /// Handler for the unbalanced reduction when removing a dust account. type DustRemoval: OnUnbalanced>; @@ -226,16 +210,14 @@ pub trait Trait: frame_system::Trait { /// The minimum amount required to keep an account open. type ExistentialDeposit: Get; - /// The fee required to create an account. - type CreationFee: Get; + /// The means of storing the balances of an account. + type AccountStore: StoredMap>; } impl, I: Instance> Subtrait for T { type Balance = T::Balance; - type OnReapAccount = T::OnReapAccount; - type OnNewAccount = T::OnNewAccount; type ExistentialDeposit = T::ExistentialDeposit; - type CreationFee = T::CreationFee; + type AccountStore = T::AccountStore; } decl_event!( @@ -243,12 +225,13 @@ decl_event!( ::AccountId, >::Balance { - /// A new account was created. - NewAccount(AccountId, Balance), - /// An account was reaped. - ReapedAccount(AccountId, Balance), - /// Transfer succeeded (from, to, value, fees). - Transfer(AccountId, AccountId, Balance, Balance), + /// An account was created with some free balance. + Endowed(AccountId, Balance), + /// An account was removed whose balance was non-zero but below ExistentialDeposit, + /// resulting in an outright loss. + DustLost(AccountId, Balance), + /// Transfer succeeded (from, to, value). + Transfer(AccountId, AccountId, Balance), /// A balance was set by root (who, free, reserved). BalanceSet(AccountId, Balance, Balance), /// Some amount was deposited (e.g. for transaction fees). @@ -375,11 +358,9 @@ decl_storage! { /// /// NOTE: THIS MAY NEVER BE IN EXISTENCE AND YET HAVE A `total().is_zero()`. If the total /// is ever zero, then the entry *MUST* be removed. - pub Account get(fn account) - build(|config: &GenesisConfig| config.balances.iter() - .map(|&(ref who, free)| (who.clone(), AccountData { free, .. Default::default() })) - .collect::>() - ): map hasher(blake2_256) T::AccountId => AccountData; + /// + /// NOTE: This is only used in the case that this module is used to store balances. + pub Account: map hasher(blake2_256) T::AccountId => AccountData; /// Any liquidity locks on some account balances. /// NOTE: Should only be accessed when setting, changing and freeing a lock. @@ -394,12 +375,19 @@ decl_storage! { config(balances): Vec<(T::AccountId, T::Balance)>; // ^^ begin, length, amount liquid at genesis build(|config: &GenesisConfig| { + assert!( + >::ExistentialDeposit::get() > Zero::zero(), + "The existential deposit should be greater than zero." + ); for (_, balance) in &config.balances { assert!( *balance >= >::ExistentialDeposit::get(), "the balance of any account should always be more than existential deposit.", ) } + for &(ref who, free) in config.balances.iter() { + T::AccountStore::insert(who, AccountData { free, .. Default::default() }); + } }); } } @@ -411,9 +399,6 @@ decl_module! { /// The minimum amount required to keep an account open. const ExistentialDeposit: T::Balance = T::ExistentialDeposit::get(); - /// The fee required to create an account. - const CreationFee: T::Balance = T::CreationFee::get(); - fn deposit_event() = default; /// Transfer some liquid free balance to another account. @@ -479,24 +464,25 @@ decl_module! { let new_free = if wipeout { Zero::zero() } else { new_free }; let new_reserved = if wipeout { Zero::zero() } else { new_reserved }; - let old_account = Account::::get(&who); - - if new_free > old_account.free { - mem::drop(PositiveImbalance::::new(new_free - old_account.free)); - } else if new_free < old_account.free { - mem::drop(NegativeImbalance::::new(old_account.free - new_free)); - } + let (free, reserved) = Self::mutate_account(&who, |account| { + if new_free > account.free { + mem::drop(PositiveImbalance::::new(new_free - account.free)); + } else if new_free < account.free { + mem::drop(NegativeImbalance::::new(account.free - new_free)); + } - if new_reserved > old_account.reserved { - mem::drop(PositiveImbalance::::new(new_reserved - old_account.reserved)); - } else if new_reserved < old_account.reserved { - mem::drop(NegativeImbalance::::new(old_account.reserved - new_reserved)); - } + if new_reserved > account.reserved { + mem::drop(PositiveImbalance::::new(new_reserved - account.reserved)); + } else if new_reserved < account.reserved { + mem::drop(NegativeImbalance::::new(account.reserved - new_reserved)); + } - let account = AccountData { free: new_free, reserved: new_reserved, ..old_account }; - Self::set_account(&who, &account, &old_account); + account.free = new_free; + account.reserved = new_reserved; - Self::deposit_event(RawEvent::BalanceSet(who, account.free, account.reserved)); + (account.free, account.reserved) + }); + Self::deposit_event(RawEvent::BalanceSet(who, free, reserved)); } /// Exactly as `transfer`, except the origin must be root and the source account may be @@ -621,87 +607,119 @@ impl, I: Instance> Module { put_storage_value(b"Balances", b"Locks", &hash, locks); put_storage_value(b"Balances", b"Account", &hash, account); } + + for (hash, balances) in StorageIterator::>::new(b"Balances", b"Account").drain() { + let nonce = take_storage_value::(b"System", b"AccountNonce", &hash).unwrap_or_default(); + let mut refs: system::RefCount = 0; + // The items in Kusama that would result in a ref count being incremented. + if have_storage_value(b"Democracy", b"Proxy", &hash) { refs += 1 } + // We skip Recovered since it's being replaced anyway. + let mut prefixed_hash = twox_64(&b":session:keys"[..]).to_vec(); + prefixed_hash.extend(&b":session:keys"[..]); + prefixed_hash.extend(&hash[..]); + if have_storage_value(b"Session", b"NextKeys", &prefixed_hash) { refs += 1 } + if have_storage_value(b"Staking", b"Bonded", &hash) { refs += 1 } + put_storage_value(b"System", b"Account", &hash, (nonce, refs, &balances)); + } } /// Get the free balance of an account. pub fn free_balance(who: impl sp_std::borrow::Borrow) -> T::Balance { - Account::::get(who.borrow()).free + Self::account(who.borrow()).free } /// Get the balance of an account that can be used for transfers, reservations, or any other /// non-locking, non-transaction-fee activity. Will be at most `free_balance`. pub fn usable_balance(who: impl sp_std::borrow::Borrow) -> T::Balance { - Account::::get(who.borrow()).usable(Reasons::Misc) + Self::account(who.borrow()).usable(Reasons::Misc) } /// Get the balance of an account that can be used for paying transaction fees (not tipping, /// or any other kind of fees, though). Will be at most `free_balance`. pub fn usable_balance_for_fees(who: impl sp_std::borrow::Borrow) -> T::Balance { - Account::::get(who.borrow()).usable(Reasons::Fee) + Self::account(who.borrow()).usable(Reasons::Fee) } /// Get the reserved balance of an account. pub fn reserved_balance(who: impl sp_std::borrow::Borrow) -> T::Balance { - Account::::get(who.borrow()).reserved + Self::account(who.borrow()).reserved } - /// Set both the free and reserved balance of an account to some new value. Will enforce - /// `ExistentialDeposit` law, annulling the account as needed. - /// - /// Will return `AccountKilled` if either reserved or free are too low. - /// - /// NOTE: This assumes that `account` is the same as `Self::account(who)` except for altered - /// values of `free` and `balance`. - /// - /// NOTE: Doesn't do any preparatory work for creating a new account, so should only be used - /// when it is known that the account already exists. + /// Get both the free and reserved balances of an account. + fn account(who: &T::AccountId) -> AccountData { + T::AccountStore::get(&who) + } + + /// Places the `free` and `reserved` parts of `new` into `account`. Also does any steps needed + /// after mutating an account. This includes DustRemoval unbalancing, in the case than the `new` + /// account's total balance is non-zero but below ED. /// - /// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that - /// the caller will do this. - fn set_account( + /// Returns the final free balance, iff the account was previously of total balance zero, known + /// as its "endowment". + fn post_mutation( who: &T::AccountId, - account: &AccountData, - old: &AccountData, - ) -> UpdateBalanceOutcome { - let total = account.free + account.reserved; + new: AccountData, + ) -> Option> { + let total = new.total(); if total < T::ExistentialDeposit::get() { - T::DustRemoval::on_unbalanced(NegativeImbalance::new(total)); - if !old.total().is_zero() { - Self::reap_account(who, total); - UpdateBalanceOutcome::AccountKilled - } else { - UpdateBalanceOutcome::StillDead + if !total.is_zero() { + T::DustRemoval::on_unbalanced(NegativeImbalance::new(total)); + Self::deposit_event(RawEvent::DustLost(who.clone(), total)); } + None } else { - if old.total().is_zero() { - Self::about_to_create_account(who, account.free); - } - Account::::insert(who, account); - UpdateBalanceOutcome::Updated + Some(new) } } - /// Register a new account (with existential balance). + /// Mutate an account to some new value, or delete it entirely with `None`. Will enforce + /// `ExistentialDeposit` law, annulling the account as needed. /// - /// This just calls appropriate hooks. It doesn't (necessarily) make any state changes. - fn about_to_create_account(who: &T::AccountId, balance: T::Balance) { - T::OnNewAccount::on_new_account(&who); - Self::deposit_event(RawEvent::NewAccount(who.clone(), balance.clone())); + /// NOTE: Doesn't do any preparatory work for creating a new account, so should only be used + /// when it is known that the account already exists. + /// + /// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that + /// the caller will do this. + fn mutate_account( + who: &T::AccountId, + f: impl FnOnce(&mut AccountData) -> R + ) -> R { + Self::try_mutate_account(who, |a| -> Result { Ok(f(a)) }) + .expect("Error is infallible; qed") } - /// Unregister an account. + /// Mutate an account to some new value, or delete it entirely with `None`. Will enforce + /// `ExistentialDeposit` law, annulling the account as needed. This will do nothing if the + /// result of `f` is an `Err`. /// - /// This just removes the nonce and leaves an event. - fn reap_account(who: &T::AccountId, dust: T::Balance) { - Locks::::remove(who); - Account::::remove(who); - T::OnReapAccount::on_reap_account(who); - Self::deposit_event(RawEvent::ReapedAccount(who.clone(), dust)); + /// NOTE: Doesn't do any preparatory work for creating a new account, so should only be used + /// when it is known that the account already exists. + /// + /// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that + /// the caller will do this. + fn try_mutate_account( + who: &T::AccountId, + f: impl FnOnce(&mut AccountData) -> Result + ) -> Result { + T::AccountStore::try_mutate_exists(who, |maybe_account| { + let mut account = maybe_account.take().unwrap_or_default(); + let was_zero = account.total().is_zero(); + f(&mut account).map(move |result| { + let maybe_endowed = if was_zero { Some(account.free) } else { None }; + *maybe_account = Self::post_mutation(who, account); + (maybe_endowed, result) + }) + }).map(|(maybe_endowed, result)| { + if let Some(endowed) = maybe_endowed { + Self::deposit_event(RawEvent::Endowed(who.clone(), endowed)); + } + result + }) } /// Update the account entry for `who`, given the locks. fn update_locks(who: &T::AccountId, locks: &[BalanceLock]) { - Account::::mutate(who, |b| { + Self::mutate_account(who, |b| { b.misc_frozen = Zero::zero(); b.fee_frozen = Zero::zero(); for l in locks.iter() { @@ -713,7 +731,21 @@ impl, I: Instance> Module { } } }); - Locks::::insert(who, locks); + + let existed = Locks::::contains_key(who); + if locks.is_empty() { + Locks::::remove(who); + if existed { + // TODO: use Locks::::hashed_key + // https://github.com/paritytech/substrate/issues/4969 + system::Module::::dec_ref(who); + } + } else { + Locks::::insert(who, locks); + if !existed { + system::Module::::inc_ref(who); + } + } } } @@ -880,9 +912,8 @@ mod imbalances { // its type declaration). // This works as long as `increase_total_issuance_by` doesn't use the Imbalance // types (basically for charging fees). -// This should eventually be refactored so that the three type items that do -// depend on the Imbalance type (TransferPayment, DustRemoval) -// are placed in their own SRML module. +// This should eventually be refactored so that the type item that +// depends on the Imbalance type (DustRemoval) is placed in its own pallet. struct ElevatedTrait, I: Instance>(T, I); impl, I: Instance> Clone for ElevatedTrait { fn clone(&self) -> Self { unimplemented!() } @@ -908,16 +939,16 @@ impl, I: Instance> frame_system::Trait for ElevatedTrait { type AvailableBlockRatio = T::AvailableBlockRatio; type Version = T::Version; type ModuleToIndex = T::ModuleToIndex; + type OnNewAccount = T::OnNewAccount; + type OnKilledAccount = T::OnKilledAccount; + type AccountData = T::AccountData; } impl, I: Instance> Trait for ElevatedTrait { type Balance = T::Balance; - type OnReapAccount = T::OnReapAccount; - type OnNewAccount = T::OnNewAccount; type Event = (); - type TransferPayment = (); type DustRemoval = (); type ExistentialDeposit = T::ExistentialDeposit; - type CreationFee = T::CreationFee; + type AccountStore = T::AccountStore; } impl, I: Instance> Currency for Module where @@ -945,10 +976,6 @@ impl, I: Instance> Currency for Module where T::ExistentialDeposit::get() } - fn free_balance(who: &T::AccountId) -> Self::Balance { - Account::::get(who).free - } - // Burn funds from the total issuance, returning a positive imbalance for the amount burned. // Is a no-op if amount to be burned is zero. fn burn(mut amount: Self::Balance) -> Self::PositiveImbalance { @@ -976,6 +1003,10 @@ impl, I: Instance> Currency for Module where NegativeImbalance::new(amount) } + fn free_balance(who: &T::AccountId) -> Self::Balance { + Self::account(who).free + } + // Ensure that an account can withdraw from their free balance given any existing withdrawal // restrictions like locks and vesting balance. // Is a no-op if amount to be withdrawn is zero. @@ -991,7 +1022,7 @@ impl, I: Instance> Currency for Module where new_balance: T::Balance, ) -> DispatchResult { if amount.is_zero() { return Ok(()) } - let min_balance = Account::::get(who).frozen(reasons.into()); + let min_balance = Self::account(who).frozen(reasons.into()); ensure!(new_balance >= min_balance, Error::::LiquidityRestrictions); Ok(()) } @@ -1006,80 +1037,39 @@ impl, I: Instance> Currency for Module where ) -> DispatchResult { if value.is_zero() || transactor == dest { return Ok(()) } - let old_from_account = Self::account(transactor); - let mut from_account = old_from_account.clone(); - let old_to_account = Self::account(dest); - let mut to_account = old_to_account.clone(); - - let would_create = to_account.total().is_zero(); - let fee = if would_create { T::CreationFee::get() } else { Zero::zero() }; - let liability = value.checked_add(&fee).ok_or(Error::::Overflow)?; - - from_account.free = from_account.free.checked_sub(&liability) - .ok_or(Error::::InsufficientBalance)?; - - // NOTE: total stake being stored in the same type means that this could never overflow - // but better to be safe than sorry. - to_account.free = to_account.free.checked_add(&value).ok_or(Error::::Overflow)?; + Self::try_mutate_account(dest, |to_account| -> DispatchResult { + Self::try_mutate_account(transactor, |from_account| -> DispatchResult { + from_account.free = from_account.free.checked_sub(&value) + .ok_or(Error::::InsufficientBalance)?; - let ed = T::ExistentialDeposit::get(); - ensure!(to_account.free >= ed, Error::::ExistentialDeposit); + // NOTE: total stake being stored in the same type means that this could never overflow + // but better to be safe than sorry. + to_account.free = to_account.free.checked_add(&value).ok_or(Error::::Overflow)?; - Self::ensure_can_withdraw( - transactor, - value, - WithdrawReason::Transfer.into(), - from_account.free, - )?; + let ed = T::ExistentialDeposit::get(); + ensure!(to_account.total() >= ed, Error::::ExistentialDeposit); - let allow_death = existence_requirement == ExistenceRequirement::AllowDeath; - ensure!(allow_death || from_account.free >= ed, Error::::KeepAlive); + Self::ensure_can_withdraw( + transactor, + value, + WithdrawReason::Transfer.into(), + from_account.free, + )?; - Self::set_account(transactor, &from_account, &old_from_account); + let allow_death = existence_requirement == ExistenceRequirement::AllowDeath; + let allow_death = allow_death && system::Module::::allow_death(transactor); + ensure!(allow_death || from_account.free >= ed, Error::::KeepAlive); - // Take action on the set_account call. - // This will emit events that _resulted_ from the transfer. - Self::set_account(dest, &to_account, &old_to_account); + Ok(()) + }) + })?; // Emit transfer event. - Self::deposit_event(RawEvent::Transfer(transactor.clone(), dest.clone(), value, fee)); - - T::TransferPayment::on_unbalanced(NegativeImbalance::new(fee)); + Self::deposit_event(RawEvent::Transfer(transactor.clone(), dest.clone(), value)); Ok(()) } - // Withdraw some free balance from an account, respecting existence requirements. - // Is a no-op if value to be withdrawn is zero. - fn withdraw( - who: &T::AccountId, - value: Self::Balance, - reasons: WithdrawReasons, - liveness: ExistenceRequirement, - ) -> result::Result { - if value.is_zero() { return Ok(NegativeImbalance::zero()); } - - let old_account = Self::account(who); - let mut account = old_account.clone(); - if let Some(new_free_account) = account.free.checked_sub(&value) { - // if we need to keep the account alive... - if liveness == ExistenceRequirement::KeepAlive - // ...and it would be dead afterwards... - && new_free_account < T::ExistentialDeposit::get() - // ...yet is was alive before - && account.free >= T::ExistentialDeposit::get() - { - Err(Error::::KeepAlive)? - } - Self::ensure_can_withdraw(who, value, reasons, new_free_account)?; - account.free = new_free_account; - Self::set_account(who, &account, &old_account); - Ok(NegativeImbalance::new(value)) - } else { - Err(Error::::InsufficientBalance)? - } - } - /// Slash a target account `who`, returning the negative imbalance created and any left over /// amount that could not be slashed. /// @@ -1095,22 +1085,19 @@ impl, I: Instance> Currency for Module where ) -> (Self::NegativeImbalance, Self::Balance) { if value.is_zero() { return (NegativeImbalance::zero(), Zero::zero()) } - let old_account = Self::account(who); - let mut account = old_account.clone(); + Self::mutate_account(who, |account| { + let free_slash = cmp::min(account.free, value); + account.free -= free_slash; - let free_slash = cmp::min(account.free, value); - account.free -= free_slash; - - let remaining_slash = value - free_slash; - let result = if !remaining_slash.is_zero() { - let reserved_slash = cmp::min(account.reserved, remaining_slash); - account.reserved -= reserved_slash; - (NegativeImbalance::new(free_slash + reserved_slash), remaining_slash - reserved_slash) - } else { - (NegativeImbalance::new(value), Zero::zero()) - }; - Self::set_account(who, &account, &old_account); - result + let remaining_slash = value - free_slash; + if !remaining_slash.is_zero() { + let reserved_slash = cmp::min(account.reserved, remaining_slash); + account.reserved -= reserved_slash; + (NegativeImbalance::new(free_slash + reserved_slash), remaining_slash - reserved_slash) + } else { + (NegativeImbalance::new(value), Zero::zero()) + } + }) } /// Deposit some `value` into the free balance of an existing target account `who`. @@ -1119,16 +1106,14 @@ impl, I: Instance> Currency for Module where fn deposit_into_existing( who: &T::AccountId, value: Self::Balance - ) -> result::Result { + ) -> Result { if value.is_zero() { return Ok(PositiveImbalance::zero()) } - let old_account = Self::account(who); - let mut account = old_account.clone(); - ensure!(!account.total().is_zero(), Error::::DeadAccount); - account.free = account.free.checked_add(&value).ok_or(Error::::Overflow)?; - - Self::set_account(who, &account, &old_account); - Ok(PositiveImbalance::new(value)) + Self::try_mutate_account(who, |account| -> Result { + ensure!(!account.total().is_zero(), Error::::DeadAccount); + account.free = account.free.checked_add(&value).ok_or(Error::::Overflow)?; + Ok(PositiveImbalance::new(value)) + }) } /// Deposit some `value` into the free balance of `who`, possibly creating a new account. @@ -1143,34 +1128,58 @@ impl, I: Instance> Currency for Module where ) -> Self::PositiveImbalance { if value.is_zero() { return Self::PositiveImbalance::zero() } - let old_account = Self::account(who); - let mut account = old_account.clone(); - let ed = T::ExistentialDeposit::get(); + Self::try_mutate_account(who, |account| -> Result { + // bail if not yet created and this operation wouldn't be enough to create it. + let ed = T::ExistentialDeposit::get(); + ensure!(value >= ed || !account.total().is_zero(), Self::PositiveImbalance::zero()); + + // defensive only: overflow should never happen, however in case it does, then this + // operation is a no-op. + account.free = account.free.checked_add(&value).ok_or(Self::PositiveImbalance::zero())?; + + Ok(PositiveImbalance::new(value)) + }).unwrap_or_else(|x| x) + } + + /// Withdraw some free balance from an account, respecting existence requirements. + /// + /// Is a no-op if value to be withdrawn is zero. + fn withdraw( + who: &T::AccountId, + value: Self::Balance, + reasons: WithdrawReasons, + liveness: ExistenceRequirement, + ) -> result::Result { + if value.is_zero() { return Ok(NegativeImbalance::zero()); } + + Self::try_mutate_account(who, |account| + -> Result + { + let new_free_account = account.free.checked_sub(&value) + .ok_or(Error::::InsufficientBalance)?; - // bail if not yet created and this operation wouldn't be enough to create it. - if value < ed && account.total().is_zero() { return Self::PositiveImbalance::zero() } + // bail if we need to keep the account alive and this would kill it. + let ed = T::ExistentialDeposit::get(); + let would_be_dead = new_free_account + account.reserved < ed; + let would_kill = would_be_dead && account.free + account.reserved >= ed; + ensure!(liveness == AllowDeath || !would_kill, Error::::KeepAlive); - // defensive only: overflow should never happen, however in case it does, then this - // operation is a no-op. - account.free = match account.free.checked_add(&value) { - Some(f) => f, - None => return Self::PositiveImbalance::zero(), - }; + Self::ensure_can_withdraw(who, value, reasons, new_free_account)?; - Self::set_account(who, &account, &old_account); + account.free = new_free_account; - PositiveImbalance::new(value) + Ok(NegativeImbalance::new(value)) + }) } /// Force the new free balance of a target account `who` to some new value `balance`. - fn make_free_balance_be(who: &T::AccountId, value: Self::Balance) -> ( - SignedImbalance, - UpdateBalanceOutcome - ) { - let old_account = Self::account(who); - let mut account = old_account.clone(); - - if value < T::ExistentialDeposit::get() && account.free.is_zero() { + fn make_free_balance_be(who: &T::AccountId, value: Self::Balance) + -> SignedImbalance + { + Self::try_mutate_account(who, |account| + -> Result, ()> + { + let ed = T::ExistentialDeposit::get(); // If we're attempting to set an existing account to less than ED, then // bypass the entire operation. It's a no-op if you follow it through, but // since this is an instance where we might account for a negative imbalance @@ -1178,24 +1187,16 @@ impl, I: Instance> Currency for Module where // equal and opposite cause (returned as an Imbalance), then in the // instance that there's no other accounts on the system at all, we might // underflow the issuance and our arithmetic will be off. - return ( - SignedImbalance::Positive(Self::PositiveImbalance::zero()), - UpdateBalanceOutcome::AccountKilled, - ) - } - let imbalance = if account.free <= value { - SignedImbalance::Positive(PositiveImbalance::new(value - account.free)) - } else { - SignedImbalance::Negative(NegativeImbalance::new(account.free - value)) - }; - account.free = value; - - // If the balance is too low, then the account is reaped. - // Free balance can never be less than ED. If that happens, it gets reduced to zero - // and the account information relevant to this subsystem is deleted (i.e. the - // account is reaped). - let outcome = Self::set_account(who, &account, &old_account); - (imbalance, outcome) + ensure!(value + account.reserved >= ed || !account.total().is_zero(), ()); + + let imbalance = if account.free <= value { + SignedImbalance::Positive(PositiveImbalance::new(value - account.free)) + } else { + SignedImbalance::Negative(NegativeImbalance::new(account.free - value)) + }; + account.free = value; + Ok(imbalance) + }).unwrap_or(SignedImbalance::Positive(Self::PositiveImbalance::zero())) } } @@ -1221,18 +1222,14 @@ impl, I: Instance> ReservableCurrency for Module /// Move `value` from the free balance from `who` to their reserved balance. /// /// Is a no-op if value to be reserved is zero. - fn reserve(who: &T::AccountId, value: Self::Balance) -> result::Result<(), DispatchError> { + fn reserve(who: &T::AccountId, value: Self::Balance) -> DispatchResult { if value.is_zero() { return Ok(()) } - let old_account = Self::account(who); - let mut account = old_account.clone(); - - account.free = account.free.checked_sub(&value).ok_or(Error::::InsufficientBalance)?; - account.reserved = account.reserved.checked_add(&value).ok_or(Error::::Overflow)?; - Self::ensure_can_withdraw(who, value, WithdrawReason::Reserve.into(), account.free)?; - - Self::set_account(who, &account, &old_account); - Ok(()) + Self::try_mutate_account(who, |account| -> DispatchResult { + account.free = account.free.checked_sub(&value).ok_or(Error::::InsufficientBalance)?; + account.reserved = account.reserved.checked_add(&value).ok_or(Error::::Overflow)?; + Self::ensure_can_withdraw(who, value, WithdrawReason::Reserve.into(), account.free) + }) } /// Unreserve some funds, returning any amount that was unable to be unreserved. @@ -1241,18 +1238,14 @@ impl, I: Instance> ReservableCurrency for Module fn unreserve(who: &T::AccountId, value: Self::Balance) -> Self::Balance { if value.is_zero() { return Zero::zero() } - let old_account = Self::account(who); - let mut account = old_account.clone(); - - let actual = cmp::min(account.reserved, value); - account.reserved -= actual; - // defensive only: this can never fail since total issuance which is at least free+reserved - // fits into the same datatype. - account.free = account.free.saturating_add(actual); - - Self::set_account(who, &account, &old_account); - - value - actual + Self::mutate_account(who, |account| { + let actual = cmp::min(account.reserved, value); + account.reserved -= actual; + // defensive only: this can never fail since total issuance which is at least free+reserved + // fits into the same data type. + account.free = account.free.saturating_add(actual); + value - actual + }) } /// Slash from reserved balance, returning the negative imbalance created, @@ -1265,47 +1258,57 @@ impl, I: Instance> ReservableCurrency for Module ) -> (Self::NegativeImbalance, Self::Balance) { if value.is_zero() { return (NegativeImbalance::zero(), Zero::zero()) } - let old_account = Self::account(who); - let mut account = old_account.clone(); - - // underflow should never happen, but it if does, there's nothing to be done here. - let actual = cmp::min(account.reserved, value); - account.reserved -= actual; - - Self::set_account(who, &account, &old_account); - - (NegativeImbalance::new(actual), value - actual) + Self::mutate_account(who, |account| { + // underflow should never happen, but it if does, there's nothing to be done here. + let actual = cmp::min(account.reserved, value); + account.reserved -= actual; + (NegativeImbalance::new(actual), value - actual) + }) } - /// Move the reserved balance of one account into the free balance of another. + /// Move the reserved balance of one account into the balance of another, according to `status`. /// - /// Is a no-op if the value to be moved is zero. + /// Is a no-op if: + /// - the value to be moved is zero; or + /// - the `slashed` id equal to `beneficiary` and the `status` is `Reserved`. fn repatriate_reserved( slashed: &T::AccountId, beneficiary: &T::AccountId, value: Self::Balance, - ) -> result::Result { - if value.is_zero() { return Ok (Zero::zero()) } + status: Status, + ) -> Result { + if value.is_zero() { return Ok(Zero::zero()) } if slashed == beneficiary { - return Ok(Self::unreserve(slashed, value)); + return match status { + Status::Free => Ok(Self::unreserve(slashed, value)), + Status::Reserved => Ok(value.saturating_sub(Self::reserved_balance(slashed))), + }; } - let old_to_account = Self::account(beneficiary); - let mut to_account = old_to_account.clone(); - ensure!(!to_account.total().is_zero(), Error::::DeadAccount); - - let old_from_account = Self::account(slashed); - let mut from_account = old_from_account.clone(); - let actual = cmp::min(from_account.reserved, value); - - to_account.free = to_account.free.checked_add(&actual).ok_or(Error::::Overflow)?; - from_account.reserved -= actual; - - Self::set_account(slashed, &from_account, &old_from_account); - Self::set_account(beneficiary, &to_account, &old_to_account); + Self::try_mutate_account(beneficiary, |to_account| -> Result { + ensure!(!to_account.total().is_zero(), Error::::DeadAccount); + Self::try_mutate_account(slashed, |from_account| -> Result { + let actual = cmp::min(from_account.reserved, value); + match status { + Status::Free => to_account.free = to_account.free.checked_add(&actual).ok_or(Error::::Overflow)?, + Status::Reserved => to_account.reserved = to_account.reserved.checked_add(&actual).ok_or(Error::::Overflow)?, + } + from_account.reserved -= actual; + Ok(value - actual) + }) + }) + } +} - Ok(value - actual) +/// Implement `OnKilledAccount` to remove the local account, if using local account storage. +/// +/// NOTE: You probably won't need to use this! This only needs to be "wired in" to System module +/// if you're using the local balance storage. **If you're using the composite system account +/// storage (which is the default in most examples and tests) then there's no need.** +impl, I: Instance> OnKilledAccount for Module { + fn on_killed_account(who: &T::AccountId) { + Account::::remove(who); } } @@ -1375,12 +1378,11 @@ where } } -impl, I: Instance> IsDeadAccount for Module -where +impl, I: Instance> IsDeadAccount for Module where T::Balance: MaybeSerializeDeserialize + Debug { fn is_dead_account(who: &T::AccountId) -> bool { // this should always be exactly equivalent to `Self::account(who).total().is_zero()` - !Account::::exists(who) + !T::AccountStore::is_explicit(who) } } diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 816768901b6023d691b8b5464f2d3a44be351504..98c7c856bc88a3e74c56c64e77107bc57bacd500 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -14,644 +14,649 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Tests for the module. - -use super::*; -use mock::{Balances, ExtBuilder, Test, System, info_from_weight, CALL}; -use sp_runtime::{Fixed64, traits::{SignedExtension, BadOrigin}}; -use frame_support::{ - assert_noop, assert_ok, assert_err, - traits::{LockableCurrency, LockIdentifier, WithdrawReason, WithdrawReasons, - Currency, ReservableCurrency, ExistenceRequirement::AllowDeath} -}; -use pallet_transaction_payment::ChargeTransactionPayment; -use frame_system::RawOrigin; - -const ID_1: LockIdentifier = *b"1 "; -const ID_2: LockIdentifier = *b"2 "; - -#[test] -fn basic_locking_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - assert_eq!(Balances::free_balance(1), 10); - Balances::set_lock(ID_1, &1, 9, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 5, AllowDeath), - Error::::LiquidityRestrictions - ); - }); -} - -#[test] -fn partial_locking_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - }); -} - -#[test] -fn lock_removal_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::all()); - Balances::remove_lock(ID_1, &1); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - }); -} - -#[test] -fn lock_replacement_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::all()); - Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - }); -} - -#[test] -fn double_locking_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); - Balances::set_lock(ID_2, &1, 5, WithdrawReasons::all()); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - }); -} - -#[test] -fn combination_locking_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::none()); - Balances::set_lock(ID_2, &1, 0, WithdrawReasons::all()); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - }); -} - -#[test] -fn lock_value_extension_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - Balances::extend_lock(ID_1, &1, 2, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - Balances::extend_lock(ID_1, &1, 8, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 3, AllowDeath), - Error::::LiquidityRestrictions - ); - }); -} - -#[test] -fn lock_reasons_should_work() { - ExtBuilder::default() - .existential_deposit(1) - .monied(true) - .build() - .execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::put(Fixed64::from_natural(1)); - Balances::set_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); - assert_noop!( - >::transfer(&1, &2, 1, AllowDeath), - Error::::LiquidityRestrictions - ); - assert_noop!( - >::reserve(&1, 1), - Error::::LiquidityRestrictions - ); - assert!( as SignedExtension>::pre_dispatch( - ChargeTransactionPayment::from(1), - &1, - CALL, - info_from_weight(1), - 1, - ).is_err()); - assert!( as SignedExtension>::pre_dispatch( - ChargeTransactionPayment::from(0), - &1, - CALL, - info_from_weight(1), - 1, - ).is_ok()); - - Balances::set_lock(ID_1, &1, 10, WithdrawReason::TransactionPayment.into()); - assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); - assert_ok!(>::reserve(&1, 1)); - assert!( as SignedExtension>::pre_dispatch( - ChargeTransactionPayment::from(1), - &1, - CALL, - info_from_weight(1), - 1, - ).is_err()); - assert!( as SignedExtension>::pre_dispatch( - ChargeTransactionPayment::from(0), - &1, - CALL, - info_from_weight(1), - 1, - ).is_err()); - }); -} - -#[test] -fn lock_block_number_extension_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, 10, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - System::set_block_number(2); - Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::all()); - assert_noop!( - >::transfer(&1, &2, 3, AllowDeath), - Error::::LiquidityRestrictions - ); - }); -} - -#[test] -fn lock_reasons_extension_should_work() { - ExtBuilder::default().existential_deposit(1).monied(true).build().execute_with(|| { - Balances::set_lock(ID_1, &1, 10, WithdrawReason::Transfer.into()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::none()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - Balances::extend_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); - assert_noop!( - >::transfer(&1, &2, 6, AllowDeath), - Error::::LiquidityRestrictions - ); - }); -} - -#[test] -fn default_indexing_on_new_accounts_should_not_work2() { - ExtBuilder::default() - .existential_deposit(10) - .creation_fee(50) - .monied(true) - .build() - .execute_with(|| { - assert_eq!(Balances::is_dead_account(&5), true); // account 5 should not exist - // ext_deposit is 10, value is 9, not satisfies for ext_deposit - assert_noop!( - Balances::transfer(Some(1).into(), 5, 9), - Error::::ExistentialDeposit, - ); - assert_eq!(Balances::is_dead_account(&5), true); // account 5 should not exist - assert_eq!(Balances::free_balance(1), 100); - }); -} - -#[test] -fn reserved_balance_should_prevent_reclaim_count() { - ExtBuilder::default() - .existential_deposit(256 * 1) - .monied(true) - .build() - .execute_with(|| { - System::inc_account_nonce(&2); - assert_eq!(Balances::is_dead_account(&2), false); - assert_eq!(Balances::is_dead_account(&5), true); - assert_eq!(Balances::total_balance(&2), 256 * 20); - - assert_ok!(Balances::reserve(&2, 256 * 19 + 1)); // account 2 becomes mostly reserved - assert_eq!(Balances::free_balance(2), 255); // "free" account deleted." - assert_eq!(Balances::total_balance(&2), 256 * 20); // reserve still exists. - assert_eq!(Balances::is_dead_account(&2), false); - assert_eq!(System::account_nonce(&2), 1); - - // account 4 tries to take index 1 for account 5. - assert_ok!(Balances::transfer(Some(4).into(), 5, 256 * 1 + 0x69)); - assert_eq!(Balances::total_balance(&5), 256 * 1 + 0x69); - assert_eq!(Balances::is_dead_account(&5), false); - - assert!(Balances::slash(&2, 256 * 19 + 2).1.is_zero()); // account 2 gets slashed - // "reserve" account reduced to 255 (below ED) so account deleted - assert_eq!(Balances::total_balance(&2), 0); - assert_eq!(System::account_nonce(&2), 0); // nonce zero - assert_eq!(Balances::is_dead_account(&2), true); - - // account 4 tries to take index 1 again for account 6. - assert_ok!(Balances::transfer(Some(4).into(), 6, 256 * 1 + 0x69)); - assert_eq!(Balances::total_balance(&6), 256 * 1 + 0x69); - assert_eq!(Balances::is_dead_account(&6), false); - }); -} - - -#[test] -fn reward_should_work() { - ExtBuilder::default().monied(true).build().execute_with(|| { - assert_eq!(Balances::total_balance(&1), 10); - assert_ok!(Balances::deposit_into_existing(&1, 10).map(drop)); - assert_eq!(Balances::total_balance(&1), 20); - assert_eq!(>::get(), 120); - }); -} - -#[test] -fn dust_account_removal_should_work() { - ExtBuilder::default() - .existential_deposit(100) - .monied(true) - .build() - .execute_with(|| { - System::inc_account_nonce(&2); - assert_eq!(System::account_nonce(&2), 1); - assert_eq!(Balances::total_balance(&2), 2000); - - assert_ok!(Balances::transfer(Some(2).into(), 5, 1901)); // index 1 (account 2) becomes zombie - assert_eq!(Balances::total_balance(&2), 0); - assert_eq!(Balances::total_balance(&5), 1901); - assert_eq!(System::account_nonce(&2), 0); - }); -} - -#[test] -fn dust_account_removal_should_work2() { - ExtBuilder::default() - .existential_deposit(100) - .creation_fee(50) - .monied(true) - .build() - .execute_with(|| { - System::inc_account_nonce(&2); - assert_eq!(System::account_nonce(&2), 1); - assert_eq!(Balances::total_balance(&2), 2000); - // index 1 (account 2) becomes zombie for 256*10 + 50(fee) < 256 * 10 (ext_deposit) - assert_ok!(Balances::transfer(Some(2).into(), 5, 1851)); - assert_eq!(Balances::total_balance(&2), 0); - assert_eq!(Balances::total_balance(&5), 1851); - assert_eq!(System::account_nonce(&2), 0); - }); -} - -#[test] -fn balance_works() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 42); - assert_eq!(Balances::free_balance(1), 42); - assert_eq!(Balances::reserved_balance(1), 0); - assert_eq!(Balances::total_balance(&1), 42); - assert_eq!(Balances::free_balance(2), 0); - assert_eq!(Balances::reserved_balance(2), 0); - assert_eq!(Balances::total_balance(&2), 0); - }); -} - -#[test] -fn balance_transfer_works() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::transfer(Some(1).into(), 2, 69)); - assert_eq!(Balances::total_balance(&1), 42); - assert_eq!(Balances::total_balance(&2), 69); - }); -} - -#[test] -fn force_transfer_works() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_noop!( - Balances::force_transfer(Some(2).into(), 1, 2, 69), - BadOrigin, - ); - assert_ok!(Balances::force_transfer(RawOrigin::Root.into(), 1, 2, 69)); - assert_eq!(Balances::total_balance(&1), 42); - assert_eq!(Balances::total_balance(&2), 69); - }); -} - -#[test] -fn reserving_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - - assert_eq!(Balances::total_balance(&1), 111); - assert_eq!(Balances::free_balance(1), 111); - assert_eq!(Balances::reserved_balance(1), 0); - - assert_ok!(Balances::reserve(&1, 69)); - - assert_eq!(Balances::total_balance(&1), 111); - assert_eq!(Balances::free_balance(1), 42); - assert_eq!(Balances::reserved_balance(1), 69); - }); -} - -#[test] -fn balance_transfer_when_reserved_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 69)); - assert_noop!( - Balances::transfer(Some(1).into(), 2, 69), - Error::::InsufficientBalance, - ); - }); -} - -#[test] -fn deducting_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 69)); - assert_eq!(Balances::free_balance(1), 42); - }); -} - -#[test] -fn refunding_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 42); - let account = Balances::account(&1); - Balances::set_account(&1, &AccountData { reserved: 69, ..account }, &account); - Balances::unreserve(&1, 69); - assert_eq!(Balances::free_balance(1), 111); - assert_eq!(Balances::reserved_balance(1), 0); - }); -} - -#[test] -fn slashing_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 69)); - assert!(Balances::slash(&1, 69).1.is_zero()); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::reserved_balance(1), 42); - assert_eq!(>::get(), 42); - }); -} - -#[test] -fn slashing_incomplete_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 42); - assert_ok!(Balances::reserve(&1, 21)); - assert_eq!(Balances::slash(&1, 69).1, 27); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::reserved_balance(1), 0); - assert_eq!(>::get(), 0); - }); -} - -#[test] -fn unreserving_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 111)); - Balances::unreserve(&1, 42); - assert_eq!(Balances::reserved_balance(1), 69); - assert_eq!(Balances::free_balance(1), 42); - }); -} - -#[test] -fn slashing_reserved_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 111)); - assert_eq!(Balances::slash_reserved(&1, 42).1, 0); - assert_eq!(Balances::reserved_balance(1), 69); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(>::get(), 69); - }); -} - -#[test] -fn slashing_incomplete_reserved_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 42)); - assert_eq!(Balances::slash_reserved(&1, 69).1, 27); - assert_eq!(Balances::free_balance(1), 69); - assert_eq!(Balances::reserved_balance(1), 0); - assert_eq!(>::get(), 69); - }); -} - -#[test] -fn transferring_reserved_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 110); - let _ = Balances::deposit_creating(&2, 1); - assert_ok!(Balances::reserve(&1, 110)); - assert_ok!(Balances::repatriate_reserved(&1, &2, 41), 0); - assert_eq!(Balances::reserved_balance(1), 69); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::reserved_balance(2), 0); - assert_eq!(Balances::free_balance(2), 42); - }); -} - -#[test] -fn transferring_reserved_balance_to_nonexistent_should_fail() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 111); - assert_ok!(Balances::reserve(&1, 111)); - assert_noop!(Balances::repatriate_reserved(&1, &2, 42), Error::::DeadAccount); - }); -} - -#[test] -fn transferring_incomplete_reserved_balance_should_work() { - ExtBuilder::default().build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 110); - let _ = Balances::deposit_creating(&2, 1); - assert_ok!(Balances::reserve(&1, 41)); - assert_ok!(Balances::repatriate_reserved(&1, &2, 69), 28); - assert_eq!(Balances::reserved_balance(1), 0); - assert_eq!(Balances::free_balance(1), 69); - assert_eq!(Balances::reserved_balance(2), 0); - assert_eq!(Balances::free_balance(2), 42); - }); -} - -#[test] -fn transferring_too_high_value_should_not_panic() { - ExtBuilder::default().build().execute_with(|| { - Account::::insert(1, AccountData { free: u64::max_value(), .. Default::default() }); - Account::::insert(2, AccountData { free: 1, .. Default::default() }); - - assert_err!( - Balances::transfer(Some(1).into(), 2, u64::max_value()), - Error::::Overflow, - ); - - assert_eq!(Balances::free_balance(1), u64::max_value()); - assert_eq!(Balances::free_balance(2), 1); - }); -} - -#[test] -fn account_create_on_free_too_low_with_other() { - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 100); - assert_eq!(>::get(), 100); - - // No-op. - let _ = Balances::deposit_creating(&2, 50); - assert_eq!(Balances::free_balance(2), 0); - assert_eq!(>::get(), 100); - }) -} - - -#[test] -fn account_create_on_free_too_low() { - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { - // No-op. - let _ = Balances::deposit_creating(&2, 50); - assert_eq!(Balances::free_balance(2), 0); - assert_eq!(>::get(), 0); - }) -} - -#[test] -fn account_removal_on_free_too_low() { - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { - assert_eq!(>::get(), 0); - - // Setup two accounts with free balance above the existential threshold. - let _ = Balances::deposit_creating(&1, 110); - let _ = Balances::deposit_creating(&2, 110); - - assert_eq!(Balances::free_balance(1), 110); - assert_eq!(Balances::free_balance(2), 110); - assert_eq!(>::get(), 220); - - // Transfer funds from account 1 of such amount that after this transfer - // the balance of account 1 will be below the existential threshold. - // This should lead to the removal of all balance of this account. - assert_ok!(Balances::transfer(Some(1).into(), 2, 20)); - - // Verify free balance removal of account 1. - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::free_balance(2), 130); - - // Verify that TotalIssuance tracks balance removal when free balance is too low. - assert_eq!(>::get(), 130); - }); -} - -#[test] -fn transfer_overflow_isnt_exploitable() { - ExtBuilder::default().creation_fee(50).build().execute_with(|| { - // Craft a value that will overflow if summed with `creation_fee`. - let evil_value = u64::max_value() - 49; - - assert_err!( - Balances::transfer(Some(1).into(), 5, evil_value), - Error::::Overflow, - ); - }); -} - -#[test] -fn burn_must_work() { - ExtBuilder::default().monied(true).build().execute_with(|| { - let init_total_issuance = Balances::total_issuance(); - let imbalance = Balances::burn(10); - assert_eq!(Balances::total_issuance(), init_total_issuance - 10); - drop(imbalance); - assert_eq!(Balances::total_issuance(), init_total_issuance); - }); -} - -#[test] -fn transfer_keep_alive_works() { - ExtBuilder::default().existential_deposit(1).build().execute_with(|| { - let _ = Balances::deposit_creating(&1, 100); - assert_err!( - Balances::transfer_keep_alive(Some(1).into(), 2, 100), - Error::::KeepAlive - ); - assert_eq!(Balances::is_dead_account(&1), false); - assert_eq!(Balances::total_balance(&1), 100); - assert_eq!(Balances::total_balance(&2), 0); - }); -} - -#[test] -#[should_panic="the balance of any account should always be more than existential deposit."] -fn cannot_set_genesis_value_below_ed() { - mock::EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = 11); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let _ = GenesisConfig:: { - balances: vec![(1, 10)], - }.assimilate_storage(&mut t).unwrap(); -} - -#[test] -fn dust_moves_between_free_and_reserved() { - ExtBuilder::default() - .existential_deposit(100) - .build() - .execute_with(|| { - // Set balance to free and reserved at the existential deposit - assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 100, 0)); - // Check balance - assert_eq!(Balances::free_balance(1), 100); - assert_eq!(Balances::reserved_balance(1), 0); - - // Reserve some free balance - assert_ok!(Balances::reserve(&1, 50)); - // Check balance, the account should be ok. - assert_eq!(Balances::free_balance(1), 50); - assert_eq!(Balances::reserved_balance(1), 50); - - // Reserve the rest of the free balance - assert_ok!(Balances::reserve(&1, 50)); - // Check balance, the account should be ok. - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::reserved_balance(1), 100); - - // Unreserve everything - Balances::unreserve(&1, 100); - // Check balance, all 100 should move to free_balance - assert_eq!(Balances::free_balance(1), 100); - assert_eq!(Balances::reserved_balance(1), 0); - }); -} - -#[test] -fn account_deleted_when_just_dust() { - ExtBuilder::default() - .existential_deposit(100) - .build() - .execute_with(|| { - // Set balance to free and reserved at the existential deposit - assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 50, 50)); - // Check balance - assert_eq!(Balances::free_balance(1), 50); - assert_eq!(Balances::reserved_balance(1), 50); - - // Reserve some free balance - let _ = Balances::slash(&1, 1); - // The account should be dead. - assert!(Balances::is_dead_account(&1)); - assert_eq!(Balances::free_balance(1), 0); - assert_eq!(Balances::reserved_balance(1), 0); - }); +//! Macro for creating the tests for the module. + +#[macro_export] +macro_rules! decl_tests { + ($test:ty, $ext_builder:ty, $existential_deposit:expr) => { + + use crate::*; + use sp_runtime::{Fixed64, traits::{SignedExtension, BadOrigin}}; + use frame_support::{ + assert_noop, assert_ok, assert_err, + traits::{ + LockableCurrency, LockIdentifier, WithdrawReason, WithdrawReasons, + Currency, ReservableCurrency, ExistenceRequirement::AllowDeath, StoredMap + } + }; + use pallet_transaction_payment::ChargeTransactionPayment; + use frame_system::RawOrigin; + + const ID_1: LockIdentifier = *b"1 "; + const ID_2: LockIdentifier = *b"2 "; + + pub type System = frame_system::Module<$test>; + pub type Balances = Module<$test>; + + pub const CALL: &<$test as frame_system::Trait>::Call = &(); + + /// create a transaction info struct from weight. Handy to avoid building the whole struct. + pub fn info_from_weight(w: Weight) -> DispatchInfo { + DispatchInfo { weight: w, pays_fee: true, ..Default::default() } + } + + #[test] + fn basic_locking_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + Balances::set_lock(ID_1, &1, 9, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 5, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + }); + } + + #[test] + fn account_should_be_reaped() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + assert_ok!(>::transfer(&1, &2, 10, AllowDeath)); + assert!(!<::AccountStore as StoredMap>>::is_explicit(&1)); + }); + } + + #[test] + fn partial_locking_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + }); + } + + #[test] + fn lock_removal_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::all()); + Balances::remove_lock(ID_1, &1); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + }); + } + + #[test] + fn lock_replacement_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::all()); + Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + }); + } + + #[test] + fn double_locking_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); + Balances::set_lock(ID_2, &1, 5, WithdrawReasons::all()); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + }); + } + + #[test] + fn combination_locking_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, u64::max_value(), WithdrawReasons::none()); + Balances::set_lock(ID_2, &1, 0, WithdrawReasons::all()); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + }); + } + + #[test] + fn lock_value_extension_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, 5, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + Balances::extend_lock(ID_1, &1, 2, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + Balances::extend_lock(ID_1, &1, 8, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 3, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + }); + } + + #[test] + fn lock_reasons_should_work() { + <$ext_builder>::default() + .existential_deposit(1) + .monied(true) + .build() + .execute_with(|| { + pallet_transaction_payment::NextFeeMultiplier::put(Fixed64::from_natural(1)); + Balances::set_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); + assert_noop!( + >::transfer(&1, &2, 1, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + assert_noop!( + >::reserve(&1, 1), + Error::<$test, _>::LiquidityRestrictions + ); + assert!( as SignedExtension>::pre_dispatch( + ChargeTransactionPayment::from(1), + &1, + CALL, + info_from_weight(1), + 1, + ).is_err()); + assert!( as SignedExtension>::pre_dispatch( + ChargeTransactionPayment::from(0), + &1, + CALL, + info_from_weight(1), + 1, + ).is_ok()); + + Balances::set_lock(ID_1, &1, 10, WithdrawReason::TransactionPayment.into()); + assert_ok!(>::transfer(&1, &2, 1, AllowDeath)); + assert_ok!(>::reserve(&1, 1)); + assert!( as SignedExtension>::pre_dispatch( + ChargeTransactionPayment::from(1), + &1, + CALL, + info_from_weight(1), + 1, + ).is_err()); + assert!( as SignedExtension>::pre_dispatch( + ChargeTransactionPayment::from(0), + &1, + CALL, + info_from_weight(1), + 1, + ).is_err()); + }); + } + + #[test] + fn lock_block_number_extension_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, 10, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + System::set_block_number(2); + Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::all()); + assert_noop!( + >::transfer(&1, &2, 3, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + }); + } + + #[test] + fn lock_reasons_extension_should_work() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + Balances::set_lock(ID_1, &1, 10, WithdrawReason::Transfer.into()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + Balances::extend_lock(ID_1, &1, 10, WithdrawReasons::none()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + Balances::extend_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); + assert_noop!( + >::transfer(&1, &2, 6, AllowDeath), + Error::<$test, _>::LiquidityRestrictions + ); + }); + } + + #[test] + fn default_indexing_on_new_accounts_should_not_work2() { + <$ext_builder>::default() + .existential_deposit(10) + .monied(true) + .build() + .execute_with(|| { + assert_eq!(Balances::is_dead_account(&5), true); + // account 5 should not exist + // ext_deposit is 10, value is 9, not satisfies for ext_deposit + assert_noop!( + Balances::transfer(Some(1).into(), 5, 9), + Error::<$test, _>::ExistentialDeposit, + ); + assert_eq!(Balances::is_dead_account(&5), true); // account 5 should not exist + assert_eq!(Balances::free_balance(1), 100); + }); + } + + #[test] + fn reserved_balance_should_prevent_reclaim_count() { + <$ext_builder>::default() + .existential_deposit(256 * 1) + .monied(true) + .build() + .execute_with(|| { + System::inc_account_nonce(&2); + assert_eq!(Balances::is_dead_account(&2), false); + assert_eq!(Balances::is_dead_account(&5), true); + assert_eq!(Balances::total_balance(&2), 256 * 20); + + assert_ok!(Balances::reserve(&2, 256 * 19 + 1)); // account 2 becomes mostly reserved + assert_eq!(Balances::free_balance(2), 255); // "free" account deleted." + assert_eq!(Balances::total_balance(&2), 256 * 20); // reserve still exists. + assert_eq!(Balances::is_dead_account(&2), false); + assert_eq!(System::account_nonce(&2), 1); + + // account 4 tries to take index 1 for account 5. + assert_ok!(Balances::transfer(Some(4).into(), 5, 256 * 1 + 0x69)); + assert_eq!(Balances::total_balance(&5), 256 * 1 + 0x69); + assert_eq!(Balances::is_dead_account(&5), false); + + assert!(Balances::slash(&2, 256 * 19 + 2).1.is_zero()); // account 2 gets slashed + // "reserve" account reduced to 255 (below ED) so account deleted + assert_eq!(Balances::total_balance(&2), 0); + assert_eq!(System::account_nonce(&2), 0); // nonce zero + assert_eq!(Balances::is_dead_account(&2), true); + + // account 4 tries to take index 1 again for account 6. + assert_ok!(Balances::transfer(Some(4).into(), 6, 256 * 1 + 0x69)); + assert_eq!(Balances::total_balance(&6), 256 * 1 + 0x69); + assert_eq!(Balances::is_dead_account(&6), false); + }); + } + + #[test] + fn reward_should_work() { + <$ext_builder>::default().monied(true).build().execute_with(|| { + assert_eq!(Balances::total_balance(&1), 10); + assert_ok!(Balances::deposit_into_existing(&1, 10).map(drop)); + assert_eq!(Balances::total_balance(&1), 20); + assert_eq!(>::get(), 120); + }); + } + + #[test] + fn dust_account_removal_should_work() { + <$ext_builder>::default() + .existential_deposit(100) + .monied(true) + .build() + .execute_with(|| { + System::inc_account_nonce(&2); + assert_eq!(System::account_nonce(&2), 1); + assert_eq!(Balances::total_balance(&2), 2000); + // index 1 (account 2) becomes zombie + assert_ok!(Balances::transfer(Some(2).into(), 5, 1901)); + assert_eq!(Balances::total_balance(&2), 0); + assert_eq!(Balances::total_balance(&5), 1901); + assert_eq!(System::account_nonce(&2), 0); + }); + } + + #[test] + fn balance_works() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 42); + assert_eq!(Balances::free_balance(1), 42); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(Balances::total_balance(&1), 42); + assert_eq!(Balances::free_balance(2), 0); + assert_eq!(Balances::reserved_balance(2), 0); + assert_eq!(Balances::total_balance(&2), 0); + }); + } + + #[test] + fn balance_transfer_works() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::transfer(Some(1).into(), 2, 69)); + assert_eq!(Balances::total_balance(&1), 42); + assert_eq!(Balances::total_balance(&2), 69); + }); + } + + #[test] + fn force_transfer_works() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_noop!( + Balances::force_transfer(Some(2).into(), 1, 2, 69), + BadOrigin, + ); + assert_ok!(Balances::force_transfer(RawOrigin::Root.into(), 1, 2, 69)); + assert_eq!(Balances::total_balance(&1), 42); + assert_eq!(Balances::total_balance(&2), 69); + }); + } + + #[test] + fn reserving_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + + assert_eq!(Balances::total_balance(&1), 111); + assert_eq!(Balances::free_balance(1), 111); + assert_eq!(Balances::reserved_balance(1), 0); + + assert_ok!(Balances::reserve(&1, 69)); + + assert_eq!(Balances::total_balance(&1), 111); + assert_eq!(Balances::free_balance(1), 42); + assert_eq!(Balances::reserved_balance(1), 69); + }); + } + + #[test] + fn balance_transfer_when_reserved_should_not_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 69)); + assert_noop!( + Balances::transfer(Some(1).into(), 2, 69), + Error::<$test, _>::InsufficientBalance, + ); + }); + } + + #[test] + fn deducting_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 69)); + assert_eq!(Balances::free_balance(1), 42); + }); + } + + #[test] + fn refunding_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 42); + Balances::mutate_account(&1, |a| a.reserved = 69); + Balances::unreserve(&1, 69); + assert_eq!(Balances::free_balance(1), 111); + assert_eq!(Balances::reserved_balance(1), 0); + }); + } + + #[test] + fn slashing_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 69)); + assert!(Balances::slash(&1, 69).1.is_zero()); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(1), 42); + assert_eq!(>::get(), 42); + }); + } + + #[test] + fn slashing_incomplete_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 42); + assert_ok!(Balances::reserve(&1, 21)); + assert_eq!(Balances::slash(&1, 69).1, 27); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(>::get(), 0); + }); + } + + #[test] + fn unreserving_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 111)); + Balances::unreserve(&1, 42); + assert_eq!(Balances::reserved_balance(1), 69); + assert_eq!(Balances::free_balance(1), 42); + }); + } + + #[test] + fn slashing_reserved_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 111)); + assert_eq!(Balances::slash_reserved(&1, 42).1, 0); + assert_eq!(Balances::reserved_balance(1), 69); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(>::get(), 69); + }); + } + + #[test] + fn slashing_incomplete_reserved_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 42)); + assert_eq!(Balances::slash_reserved(&1, 69).1, 27); + assert_eq!(Balances::free_balance(1), 69); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(>::get(), 69); + }); + } + + #[test] + fn repatriating_reserved_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 110); + let _ = Balances::deposit_creating(&2, 1); + assert_ok!(Balances::reserve(&1, 110)); + assert_ok!(Balances::repatriate_reserved(&1, &2, 41, Status::Free), 0); + assert_eq!(Balances::reserved_balance(1), 69); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(2), 0); + assert_eq!(Balances::free_balance(2), 42); + }); + } + + #[test] + fn transferring_reserved_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 110); + let _ = Balances::deposit_creating(&2, 1); + assert_ok!(Balances::reserve(&1, 110)); + assert_ok!(Balances::repatriate_reserved(&1, &2, 41, Status::Reserved), 0); + assert_eq!(Balances::reserved_balance(1), 69); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(2), 41); + assert_eq!(Balances::free_balance(2), 1); + }); + } + + #[test] + fn transferring_reserved_balance_to_nonexistent_should_fail() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(Balances::reserve(&1, 111)); + assert_noop!(Balances::repatriate_reserved(&1, &2, 42, Status::Free), Error::<$test, _>::DeadAccount); + }); + } + + #[test] + fn transferring_incomplete_reserved_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 110); + let _ = Balances::deposit_creating(&2, 1); + assert_ok!(Balances::reserve(&1, 41)); + assert_ok!(Balances::repatriate_reserved(&1, &2, 69, Status::Free), 28); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(Balances::free_balance(1), 69); + assert_eq!(Balances::reserved_balance(2), 0); + assert_eq!(Balances::free_balance(2), 42); + }); + } + + #[test] + fn transferring_too_high_value_should_not_panic() { + <$ext_builder>::default().build().execute_with(|| { + Balances::make_free_balance_be(&1, u64::max_value()); + Balances::make_free_balance_be(&2, 1); + + assert_err!( + Balances::transfer(Some(1).into(), 2, u64::max_value()), + Error::<$test, _>::Overflow, + ); + + assert_eq!(Balances::free_balance(1), u64::max_value()); + assert_eq!(Balances::free_balance(2), 1); + }); + } + + #[test] + fn account_create_on_free_too_low_with_other() { + <$ext_builder>::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 100); + assert_eq!(>::get(), 100); + + // No-op. + let _ = Balances::deposit_creating(&2, 50); + assert_eq!(Balances::free_balance(2), 0); + assert_eq!(>::get(), 100); + }) + } + + #[test] + fn account_create_on_free_too_low() { + <$ext_builder>::default().existential_deposit(100).build().execute_with(|| { + // No-op. + let _ = Balances::deposit_creating(&2, 50); + assert_eq!(Balances::free_balance(2), 0); + assert_eq!(>::get(), 0); + }) + } + + #[test] + fn account_removal_on_free_too_low() { + <$ext_builder>::default().existential_deposit(100).build().execute_with(|| { + assert_eq!(>::get(), 0); + + // Setup two accounts with free balance above the existential threshold. + let _ = Balances::deposit_creating(&1, 110); + let _ = Balances::deposit_creating(&2, 110); + + assert_eq!(Balances::free_balance(1), 110); + assert_eq!(Balances::free_balance(2), 110); + assert_eq!(>::get(), 220); + + // Transfer funds from account 1 of such amount that after this transfer + // the balance of account 1 will be below the existential threshold. + // This should lead to the removal of all balance of this account. + assert_ok!(Balances::transfer(Some(1).into(), 2, 20)); + + // Verify free balance removal of account 1. + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::free_balance(2), 130); + + // Verify that TotalIssuance tracks balance removal when free balance is too low. + assert_eq!(>::get(), 130); + }); + } + + #[test] + fn burn_must_work() { + <$ext_builder>::default().monied(true).build().execute_with(|| { + let init_total_issuance = Balances::total_issuance(); + let imbalance = Balances::burn(10); + assert_eq!(Balances::total_issuance(), init_total_issuance - 10); + drop(imbalance); + assert_eq!(Balances::total_issuance(), init_total_issuance); + }); + } + + #[test] + fn transfer_keep_alive_works() { + <$ext_builder>::default().existential_deposit(1).build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 100); + assert_noop!( + Balances::transfer_keep_alive(Some(1).into(), 2, 100), + Error::<$test, _>::KeepAlive + ); + assert_eq!(Balances::is_dead_account(&1), false); + assert_eq!(Balances::total_balance(&1), 100); + assert_eq!(Balances::total_balance(&2), 0); + }); + } + + #[test] + #[should_panic = "the balance of any account should always be more than existential deposit."] + fn cannot_set_genesis_value_below_ed() { + ($existential_deposit).with(|v| *v.borrow_mut() = 11); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>().unwrap(); + let _ = GenesisConfig::<$test> { + balances: vec![(1, 10)], + }.assimilate_storage(&mut t).unwrap(); + } + + #[test] + fn dust_moves_between_free_and_reserved() { + <$ext_builder>::default() + .existential_deposit(100) + .build() + .execute_with(|| { + // Set balance to free and reserved at the existential deposit + assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 100, 0)); + // Check balance + assert_eq!(Balances::free_balance(1), 100); + assert_eq!(Balances::reserved_balance(1), 0); + + // Reserve some free balance + assert_ok!(Balances::reserve(&1, 50)); + // Check balance, the account should be ok. + assert_eq!(Balances::free_balance(1), 50); + assert_eq!(Balances::reserved_balance(1), 50); + + // Reserve the rest of the free balance + assert_ok!(Balances::reserve(&1, 50)); + // Check balance, the account should be ok. + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(1), 100); + + // Unreserve everything + Balances::unreserve(&1, 100); + // Check balance, all 100 should move to free_balance + assert_eq!(Balances::free_balance(1), 100); + assert_eq!(Balances::reserved_balance(1), 0); + }); + } + + #[test] + fn account_deleted_when_just_dust() { + <$ext_builder>::default() + .existential_deposit(100) + .build() + .execute_with(|| { + // Set balance to free and reserved at the existential deposit + assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 50, 50)); + // Check balance + assert_eq!(Balances::free_balance(1), 50); + assert_eq!(Balances::reserved_balance(1), 50); + + // Reserve some free balance + let _ = Balances::slash(&1, 1); + // The account should be dead. + assert!(Balances::is_dead_account(&1)); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(1), 0); + }); + } + } } diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs new file mode 100644 index 0000000000000000000000000000000000000000..3a5c2178f88cdc2fb8a7f74bc4a78a705588edcc --- /dev/null +++ b/frame/balances/src/tests_composite.rs @@ -0,0 +1,136 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Test utilities + +use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header}; +use sp_core::H256; +use sp_io; +use frame_support::{impl_outer_origin, parameter_types}; +use frame_support::traits::Get; +use frame_support::weights::{Weight, DispatchInfo}; +use std::cell::RefCell; +use crate::{GenesisConfig, Module, Trait, decl_tests}; + +use frame_system as system; +impl_outer_origin!{ + pub enum Origin for Test {} +} + +thread_local! { + static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); +} + +pub struct ExistentialDeposit; +impl Get for ExistentialDeposit { + fn get() -> u64 { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow()) } +} + +// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = (); + type Hash = H256; + type Hashing = ::sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = super::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); +} +parameter_types! { + pub const TransactionBaseFee: u64 = 0; + pub const TransactionByteFee: u64 = 1; +} +impl pallet_transaction_payment::Trait for Test { + type Currency = Module; + type OnTransactionPayment = (); + type TransactionBaseFee = TransactionBaseFee; + type TransactionByteFee = TransactionByteFee; + type WeightToFee = ConvertInto; + type FeeMultiplierUpdate = (); +} +impl Trait for Test { + type Balance = u64; + type DustRemoval = (); + type Event = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = system::Module; +} + +pub struct ExtBuilder { + existential_deposit: u64, + monied: bool, +} +impl Default for ExtBuilder { + fn default() -> Self { + Self { + existential_deposit: 1, + monied: false, + } + } +} +impl ExtBuilder { + pub fn existential_deposit(mut self, existential_deposit: u64) -> Self { + self.existential_deposit = existential_deposit; + self + } + pub fn monied(mut self, monied: bool) -> Self { + self.monied = monied; + self + } + pub fn set_associated_consts(&self) { + EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); + } + pub fn build(self) -> sp_io::TestExternalities { + self.set_associated_consts(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + GenesisConfig:: { + balances: if self.monied { + vec![ + (1, 10 * self.existential_deposit), + (2, 20 * self.existential_deposit), + (3, 30 * self.existential_deposit), + (4, 40 * self.existential_deposit), + (12, 10 * self.existential_deposit) + ] + } else { + vec![] + }, + }.assimilate_storage(&mut t).unwrap(); + t.into() + } +} + +decl_tests!{ Test, ExtBuilder, EXISTENTIAL_DEPOSIT } diff --git a/frame/balances/src/mock.rs b/frame/balances/src/tests_local.rs similarity index 79% rename from frame/balances/src/mock.rs rename to frame/balances/src/tests_local.rs index 5eb722733f4640db7f4ee0d2f53b2a33a15fa00d..861c1972127a0751dd4372fdec80cb8588d0e1f1 100644 --- a/frame/balances/src/mock.rs +++ b/frame/balances/src/tests_local.rs @@ -20,10 +20,10 @@ use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header use sp_core::H256; use sp_io; use frame_support::{impl_outer_origin, parameter_types}; -use frame_support::traits::Get; +use frame_support::traits::{Get, StorageMapShim}; use frame_support::weights::{Weight, DispatchInfo}; use std::cell::RefCell; -use crate::{GenesisConfig, Module, Trait}; +use crate::{GenesisConfig, Module, Trait, decl_tests}; use frame_system as system; impl_outer_origin!{ @@ -31,8 +31,7 @@ impl_outer_origin!{ } thread_local! { - pub(crate) static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); - static CREATION_FEE: RefCell = RefCell::new(0); + static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); } pub struct ExistentialDeposit; @@ -40,11 +39,6 @@ impl Get for ExistentialDeposit { fn get() -> u64 { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow()) } } -pub struct CreationFee; -impl Get for CreationFee { - fn get() -> u64 { CREATION_FEE.with(|v| *v.borrow()) } -} - // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. #[derive(Clone, PartialEq, Eq, Debug)] pub struct Test; @@ -71,6 +65,9 @@ impl frame_system::Trait for Test { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = super::AccountData; + type OnNewAccount = (); + type OnKilledAccount = Module; } parameter_types! { pub const TransactionBaseFee: u64 = 0; @@ -86,25 +83,25 @@ impl pallet_transaction_payment::Trait for Test { } impl Trait for Test { type Balance = u64; - type OnReapAccount = System; - type OnNewAccount = (); - type Event = (); type DustRemoval = (); - type TransferPayment = (); + type Event = (); type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = StorageMapShim< + super::Account, + system::CallOnCreatedAccount, + system::CallKillAccount, + u64, super::AccountData + >; } pub struct ExtBuilder { existential_deposit: u64, - creation_fee: u64, monied: bool, } impl Default for ExtBuilder { fn default() -> Self { Self { - existential_deposit: 0, - creation_fee: 0, + existential_deposit: 1, monied: false, } } @@ -114,10 +111,6 @@ impl ExtBuilder { self.existential_deposit = existential_deposit; self } - pub fn creation_fee(mut self, creation_fee: u64) -> Self { - self.creation_fee = creation_fee; - self - } pub fn monied(mut self, monied: bool) -> Self { self.monied = monied; if self.existential_deposit == 0 { @@ -127,7 +120,6 @@ impl ExtBuilder { } pub fn set_associated_consts(&self) { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - CREATION_FEE.with(|v| *v.borrow_mut() = self.creation_fee); } pub fn build(self) -> sp_io::TestExternalities { self.set_associated_consts(); @@ -149,12 +141,4 @@ impl ExtBuilder { } } -pub type System = frame_system::Module; -pub type Balances = Module; - -pub const CALL: &::Call = &(); - -/// create a transaction info struct from weight. Handy to avoid building the whole struct. -pub fn info_from_weight(w: Weight) -> DispatchInfo { - DispatchInfo { weight: w, pays_fee: true, ..Default::default() } -} +decl_tests!{ Test, ExtBuilder, EXISTENTIAL_DEPOSIT } diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..72395d5cb1463599f9d67d0126bd8e5d091a833e --- /dev/null +++ b/frame/benchmarking/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "frame-benchmarking" +version = "2.0.0-alpha.1" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.1.2", default-features = false } +sp-api = { version = "2.0.0-alpha.1", path = "../../primitives/api", default-features = false } +sp-runtime-interface = { version = "2.0.0-alpha.1", path = "../../primitives/runtime-interface", default-features = false } +sp-std = { version = "2.0.0-alpha.1", path = "../../primitives/std", default-features = false } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.1"} + +[features] +default = [ "std" ] +std = [ "sp-runtime-interface/std", "sp-api/std", "codec/std", "sp-std/std" ] diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..e6b6a4f3f59eca79f0d6f428897e6645bbc31383 --- /dev/null +++ b/frame/benchmarking/src/lib.rs @@ -0,0 +1,472 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Macro for benchmarking a FRAME runtime. + +#![cfg_attr(not(feature = "std"), no_std)] + +mod utils; +pub use utils::*; +#[doc(hidden)] +pub use sp_io::storage::root as storage_root; + +/// Construct pallet benchmarks for weighing dispatchables. +/// +/// Works around the idea of complexity parameters, named by a single letter (which is usually +/// upper cased in complexity notation but is lower-cased for use in this macro). +/// +/// Complexity parameters ("parameters") have a range which is a `u32` pair. Every time a benchmark +/// is prepared and run, this parameter takes a concrete value within the range. There is an +/// associated instancing block, which is a single expression that is evaluated during +/// preparation. It may use `?` (`i.e. `return Err(...)`) to bail with a string error. Here's a +/// few examples: +/// +/// ```ignore +/// // These two are equivalent: +/// let x in 0 .. 10; +/// let x in 0 .. 10 => (); +/// // This one calls a setup function and might return an error (which would be terminal). +/// let y in 0 .. 10 => setup(y)?; +/// // This one uses a code block to do lots of stuff: +/// let z in 0 .. 10 => { +/// let a = z * z / 5; +/// let b = do_something(a)?; +/// combine_into(z, b); +/// } +/// ``` +/// +/// Note that due to parsing restrictions, if the `from` expression is not a single token (i.e. a +/// literal or constant), then it must be parenthesised. +/// +/// The macro allows for a number of "arms", each representing an individual benchmark. Using the +/// simple syntax, the associated dispatchable function maps 1:1 with the benchmark and the name of +/// the benchmark is the same as that of the associated function. However, extended syntax allows +/// for arbitrary expresions to be evaluated in a benchmark (including for example, +/// `on_initialize`). +/// +/// The macro allows for common parameters whose ranges and instancing expressions may be drawn upon +/// (or not) by each arm. Syntax is available to allow for only the range to be drawn upon if +/// desired, allowing an alternative instancing expression to be given. +/// +/// Each arm may also have a block of code which is run prior to any instancing and a block of code +/// which is run afterwards. All code blocks may draw upon the specific value of each parameter +/// at any time. Local variables are shared between the two pre- and post- code blocks, but do not +/// leak from the interior of any instancing expressions. +/// +/// Any common parameters that are unused in an arm do not have their instancing expressions +/// evaluated. +/// +/// Example: +/// ```ignore +/// benchmarks! { +/// // common parameter; just one for this example. +/// _ { +/// let l in 1 .. MAX_LENGTH => initialize_l(l); +/// } +/// +/// // first dispatchable: foo; this is a user dispatchable and operates on a `u8` vector of +/// // size `l`, which we allow to be initialised as usual. +/// foo { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let l = ...; +/// } _(Origin::Signed(caller), vec![0u8; l]) +/// +/// // second dispatchable: bar; this is a root dispatchable and accepts a `u8` vector of size +/// // `l`. We don't want it preininitialised like before so we override using the `=> ()` +/// // notation. +/// // In this case, we explicitly name the call using `bar` instead of `_`. +/// bar { +/// let l = _ .. _ => (); +/// } bar(Origin::Root, vec![0u8; l]) +/// +/// // third dispatchable: baz; this is a user dispatchable. It isn't dependent on length like the +/// // other two but has its own complexity `c` that needs setting up. It uses `caller` (in the +/// // pre-instancing block) within the code block. This is only allowed in the param instancers +/// // of arms. Instancers of common params cannot optimistically draw upon hypothetical variables +/// // that the arm's pre-instancing code block might have declared. +/// baz1 { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let c = 0 .. 10 => setup_c(&caller, c); +/// } baz(Origin::Signed(caller)) +/// +/// // this is a second benchmark of the baz dispatchable with a different setup. +/// baz2 { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let c = 0 .. 10 => setup_c_in_some_other_way(&caller, c); +/// } baz(Origin::Signed(caller)) +/// +/// // this is benchmarking some code that is not a dispatchable. +/// populate_a_set { +/// let x in 0 .. 10_000; +/// let mut m = Vec::::new(); +/// for i in 0..x { +/// m.insert(i); +/// } +/// } { m.into_iter().collect::() } +/// } +/// ``` +#[macro_export] +macro_rules! benchmarks { + ( + _ { + $( + let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr; + )* + } + $( $rest:tt )* + ) => { + $crate::benchmarks_iter!({ + $( { $common , $common_from , $common_to , $common_instancer } )* + } ( ) $( $rest )* ); + } +} + +#[macro_export] +macro_rules! impl_benchmark { + ( + $( $name:ident ),* + ) => { + impl $crate::Benchmarking<$crate::BenchmarkResults> for Module { + fn run_benchmark(extrinsic: Vec, steps: Vec, repeat: u32) -> Result, &'static str> { + // Map the input to the selected benchmark. + let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice()) + .map_err(|_| "Could not find extrinsic")?; + let selected_benchmark = match extrinsic { + $( stringify!($name) => SelectedBenchmark::$name, )* + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + $crate::benchmarking::commit_db(); + $crate::benchmarking::wipe_db(); + + let components = , RawOrigin>>::components(&selected_benchmark); + let mut results: Vec<$crate::BenchmarkResults> = Vec::new(); + + // Default number of steps for a component. + let mut prev_steps = &10; + + // Select the component we will be benchmarking. Each component will be benchmarked. + for (idx, (name, low, high)) in components.iter().enumerate() { + // Get the number of steps for this component. + let steps = steps.get(idx).unwrap_or(&prev_steps); + prev_steps = steps; + + // Create up to `STEPS` steps for that component between high and low. + let step_size = ((high - low) / steps).max(1); + let num_of_steps = (high - low) / step_size + 1; + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = low + step_size * s; + + // Select the mid value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .map(|(n, l, h)| + (*n, if n == name { component_value } else { *h }) + ).collect(); + + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to benchmark. + let (call, caller) = , RawOrigin>>::instance(&selected_benchmark, &c)?; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + $crate::benchmarking::commit_db(); + // Time the extrinsic logic. + let start_extrinsic = $crate::benchmarking::current_time(); + call.dispatch(caller.into())?; + let finish_extrinsic = $crate::benchmarking::current_time(); + let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + // Time the storage root recalculation. + let start_storage_root = $crate::benchmarking::current_time(); + $crate::storage_root(); + let finish_storage_root = $crate::benchmarking::current_time(); + let elapsed_storage_root = finish_storage_root - start_storage_root; + results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + // Wipe the DB back to the genesis state. + $crate::benchmarking::wipe_db(); + } + } + } + return Ok(results); + } + } + } +} + +#[macro_export] +#[allow(missing_docs)] +macro_rules! benchmarks_iter { + // mutation arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) $( $rest )* + } + }; + // mutation arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { Ok((crate::Call::::$dispatch($($arg),*), $origin)) } $( $rest )* + } + }; + // iteration arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: { $eval:expr } + $( $rest:tt )* + ) => { + $crate::benchmark_backend! { + $name { $( $common )* } { } { $eval } { $( $code )* } + } + $crate::benchmarks_iter!( { $( $common )* } ( $( $names )* $name ) $( $rest )* ); + }; + // iteration-exit arm + ( { $( $common:tt )* } ( $( $names:ident )* ) ) => { + $crate::selected_benchmark!( $( $names ),* ); + $crate::impl_benchmark!( $( $names ),* ); + } +} + +#[macro_export] +#[allow(missing_docs)] +macro_rules! benchmark_backend { + // parsing arms + ($name:ident { + $( $common:tt )* + } { + $( PRE { $( $pre_parsed:tt )* } )* + } { $eval:expr } { + let $pre_id:tt : $pre_ty:ty = $pre_ex:expr; + $( $rest:tt )* + } ) => { + $crate::benchmark_backend! { + $name { $( $common )* } { + $( PRE { $( $pre_parsed )* } )* + PRE { $pre_id , $pre_ty , $pre_ex } + } { $eval } { $( $rest )* } + } + }; + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $param:ident in ( $param_from:expr ) .. $param_to:expr => $param_instancer:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { + $( $parsed )* + PARAM { $param , $param_from , $param_to , $param_instancer } + } { $eval } { $( $rest )* } + } + }; + // mutation arm to look after defaulting to a common param + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $param:ident in ...; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { + $( { $common , $common_from , $common_to , $common_instancer } )* + } { + $( $parsed )* + } { $eval } { + let $param + in ({ $( let $common = $common_from; )* $param }) + .. ({ $( let $common = $common_to; )* $param }) + => ({ $( let $common = || -> Result<(), &'static str> { $common_instancer ; Ok(()) }; )* $param()? }); + $( $rest )* + } + } + }; + // mutation arm to look after defaulting only the range to common param + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $param:ident in _ .. _ => $param_instancer:expr ; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { + $( { $common , $common_from , $common_to , $common_instancer } )* + } { + $( $parsed )* + } { $eval } { + let $param + in ({ $( let $common = $common_from; )* $param }) + .. ({ $( let $common = $common_to; )* $param }) + => $param_instancer ; + $( $rest )* + } + } + }; + // mutation arm to look after a single tt for param_from. + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $param:ident in $param_from:tt .. $param_to:expr => $param_instancer:expr ; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $param in ( $param_from ) .. $param_to => $param_instancer; + $( $rest )* + } + } + }; + // mutation arm to look after the default tail of `=> ()` + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $param:ident in $param_from:tt .. $param_to:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $param in $param_from .. $param_to => (); + $( $rest )* + } + } + }; + // mutation arm to look after `let _ =` + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:expr } { + let $pre_id:tt = $pre_ex:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $pre_id : _ = $pre_ex; + $( $rest )* + } + } + }; + // actioning arm + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* + $( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )* + } { $eval:expr } { $( $post:tt )* } ) => { + #[allow(non_camel_case_types)] + struct $name; + #[allow(unused_variables)] + impl $crate::BenchmarkingSetup, RawOrigin> for $name { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + vec! [ + $( + ($crate::BenchmarkParameter::$param, $param_from, $param_to) + ),* + ] + } + + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + $( + let $common = $common_from; + )* + $( + // Prepare instance + let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + )* + $( + let $pre_id : $pre_ty = $pre_ex; + )* + $( $param_instancer ; )* + $( $post )* + $eval + } + } + } +} + +/// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. +/// +/// Every variant must implement [`BenchmarkingSetup`]. +/// +/// ```nocompile +/// +/// struct Transfer; +/// impl BenchmarkingSetup for Transfer { ... } +/// +/// struct SetBalance; +/// impl BenchmarkingSetup for SetBalance { ... } +/// +/// selected_benchmark!(Transfer, SetBalance); +/// ``` +#[macro_export] +macro_rules! selected_benchmark { + ( + $( $bench:ident ),* + ) => { + // The list of available benchmarks for this pallet. + #[allow(non_camel_case_types)] + enum SelectedBenchmark { + $( $bench, )* + } + + // Allow us to select a benchmark from the list of available benchmarks. + impl $crate::BenchmarkingSetup, RawOrigin> for SelectedBenchmark { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetup< + T, + Call, + RawOrigin, + >>::components(&$bench), )* + } + } + + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result<(Call, RawOrigin), &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetup< + T, + Call, + RawOrigin, + >>::instance(&$bench, components), )* + } + } + } + }; +} diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs new file mode 100644 index 0000000000000000000000000000000000000000..9db981a61c84133576e6ae2fcab9244d2c95ebdc --- /dev/null +++ b/frame/benchmarking/src/utils.rs @@ -0,0 +1,97 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Interfaces, types and utils for benchmarking a FRAME runtime. + +use codec::{Encode, Decode}; +use sp_std::vec::Vec; +use sp_io::hashing::blake2_256; + +/// An alphabet of possible parameters to use for benchmarking. +#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)] +#[allow(missing_docs)] +#[allow(non_camel_case_types)] +pub enum BenchmarkParameter { + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, +} + +/// Results from running benchmarks on a FRAME pallet. +/// Contains duration of the function call in nanoseconds along with the benchmark parameters +/// used for that benchmark result. +pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128, u128); + +sp_api::decl_runtime_apis! { + /// Runtime api for benchmarking a FRAME runtime. + pub trait Benchmark { + /// Dispatch the given benchmark. + fn dispatch_benchmark( + module: Vec, + extrinsic: Vec, + steps: Vec, + repeat: u32, + ) -> Option>; + } +} + +/// Interface that provides functions for benchmarking the runtime. +#[sp_runtime_interface::runtime_interface] +pub trait Benchmarking { + /// Get the number of nanoseconds passed since the UNIX epoch + /// + /// WARNING! This is a non-deterministic call. Do not use this within + /// consensus critical logic. + fn current_time() -> u128 { + std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("Unix time doesn't go backwards; qed") + .as_nanos() + } + + /// Reset the trie database to the genesis state. + fn wipe_db(&mut self) { + self.wipe() + } + + /// Commit pending storage changes to the trie database and clear the database cache. + fn commit_db(&mut self) { + self.commit() + } +} + +/// The pallet benchmarking trait. +pub trait Benchmarking { + /// Run the benchmarks for this pallet. + /// + /// Parameters + /// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes. + /// - `steps`: The number of sample points you want to take across the range of parameters. + /// - `repeat`: The number of times you want to repeat a benchmark. + fn run_benchmark(extrinsic: Vec, steps: Vec, repeat: u32) -> Result, &'static str>; +} + +/// The required setup for creating a benchmark. +pub trait BenchmarkingSetup { + /// Return the components and their ranges which should be tested in this benchmark. + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; + + /// Set up the storage, and prepare a call and caller to test in a single run of the benchmark. + fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result<(Call, RawOrigin), &'static str>; +} + +/// Grab an account, seeded by a name and index. +pub fn account(name: &'static str, index: u32, seed: u32) -> AccountId { + let entropy = (name, index, seed).using_encoded(blake2_256); + AccountId::decode(&mut &entropy[..]).unwrap_or_default() +} diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index 8f7ffa76534908320aeed2761ef09b3d9d5a789c..ca2f3203d5a800afe2b0118add8e6dc72cbed2da 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "pallet-collective" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../balances" } [features] default = ["std"] diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 8d811b8dec2e7045f1eb819f7dee35e2a959edbf..605cda3cb77e22c4e2aa28cd02ac2f52dac8834d 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . //! Collective system: Members of a set of account IDs can make their collective feelings known -//! through dispatched calls from one of two specialised origins. +//! through dispatched calls from one of two specialized origins. //! //! The membership can be provided in one of two ways: either directly, using the Root-dispatchable //! function `set_members`, or indirectly, through implementing the `ChangeMembers` @@ -191,7 +191,7 @@ decl_module! { let proposal_hash = T::Hashing::hash_of(&proposal); - ensure!(!>::exists(proposal_hash), Error::::DuplicateProposal); + ensure!(!>::contains_key(proposal_hash), Error::::DuplicateProposal); if threshold < 2 { let seats = Self::members().len() as MemberCount; @@ -433,6 +433,9 @@ mod tests { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Origin = Origin; @@ -454,7 +457,7 @@ mod tests { NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Module, Call, Event}, + System: system::{Module, Call, Event}, Collective: collective::::{Module, Call, Event, Origin, Config}, DefaultCollective: collective::{Module, Call, Event, Origin, Config}, } diff --git a/frame/contracts/COMPLEXITY.md b/frame/contracts/COMPLEXITY.md index a29127f778f7ebc37f048a2bcb302e081385564f..e44d3006c8ed363e63909f9c8504545653f71d71 100644 --- a/frame/contracts/COMPLEXITY.md +++ b/frame/contracts/COMPLEXITY.md @@ -141,7 +141,7 @@ Note that in case of storage modification we need to construct a key in the unde - then perform `blake2_256` hashing of the storage key. - concatenation of these hashes will constitute the key in the underlying storage. -There is also a special case to think of: if the balance of some account goes below `existential_deposit`, then all storage entries of that account will be erased, which requires time proprotional to the number of storage entries that account has. +There is also a special case to think of: if the balance of some account goes below `existential_deposit`, then all storage entries of that account will be erased, which requires time proportional to the number of storage entries that account has. **complexity**: `N` inserts into a `Map` or eventually into the storage (if committed). Every deleted account will induce removal of all its storage which is proportional to the number of storage entries that account has. @@ -236,7 +236,7 @@ This function takes the code of the constructor and input data. Instantiation of **Note** that the complexity of executing the constructor code should be considered separately. -**Note** that the complexity of `DetermineContractAddress` hook should be considered separately as well. Most likely it will use some kind of hashing over the code of the constructor and input data. The default `SimpleAddressDeterminator` does precisely that. +**Note** that the complexity of `DetermineContractAddress` hook should be considered separately as well. Most likely it will use some kind of hashing over the code of the constructor and input data. The default `SimpleAddressDeterminer` does precisely that. **Note** that the constructor returns code in the owned form and it's obtained via return facilities, which should have take fee for the return value. diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index 159e9f9d0c199464d13f044b4ea92a475a38323a..6feaf59f573fb96cdcc25340fde0544388248dd0 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "pallet-contracts" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } @@ -11,22 +13,22 @@ pwasm-utils = { version = "0.12.0", default-features = false } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } parity-wasm = { version = "0.41.0", default-features = false } wasmi-validation = { version = "0.3.0", default-features = false } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-sandbox = { version = "0.8.0", default-features = false, path = "../../primitives/sandbox" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "common" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-sandbox = { version = "0.8.0-alpha.1", default-features = false, path = "../../primitives/sandbox" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +pallet-contracts-primitives = { version = "2.0.0-alpha.1", default-features = false, path = "common" } [dev-dependencies] wabt = "0.9.2" assert_matches = "1.3.0" hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } -pallet-timestamp = { version = "2.0.0", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "2.0.0", path = "../randomness-collective-flip" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../balances" } +pallet-timestamp = { version = "2.0.0-alpha.1", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "2.0.0-alpha.1", path = "../randomness-collective-flip" } [features] default = ["std"] diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index 6e4ee050bd1978b3da839e5f2d07a68192d05390..03a03038d817b9fd55c54c41f12b713311dcd4ef 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -1,14 +1,17 @@ [package] name = "pallet-contracts-primitives" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] # This crate should not rely on any of the frame primitives. codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] diff --git a/frame/contracts/common/src/lib.rs b/frame/contracts/common/src/lib.rs index e54b4c8d55393b6e23ba06a52776ac48a3ad13cc..6a74a417fa0fe30040297de90a52805ba3a15855 100644 --- a/frame/contracts/common/src/lib.rs +++ b/frame/contracts/common/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! A crate that hosts a common definitions that are relevent for the pallet-contracts. +//! A crate that hosts a common definitions that are relevant for the pallet-contracts. #![cfg_attr(not(feature = "std"), no_std)] diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index d59260d11f536da807679531e4350edaf9d4e8d8..b91e17e608a1122b69e5953ef506d48ff6ef164e 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "pallet-contracts-rpc" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.3" jsonrpc-derive = "14.0.3" -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0", path = "../../../primitives/rpc" } +sp-blockchain = { version = "2.0.0-alpha.1", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.1", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-alpha.1", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -pallet-contracts-primitives = { version = "2.0.0", path = "../common" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0", path = "./runtime-api" } +sp-runtime = { version = "2.0.0-alpha.1", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.1", path = "../../../primitives/api" } +pallet-contracts-primitives = { version = "2.0.0-alpha.1", path = "../common" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.1", path = "./runtime-api" } [dev-dependencies] serde_json = "1.0.41" diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index dad9b92f6a30891b30f5307a3d732b87f05b9f49..c84ab739df1cc3c26333eb7848cb378496cf7fd4 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0" +version = "0.8.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-api = { version = "2.0.0", default-features = false, path = "../../../../primitives/api" } +sp-api = { version = "2.0.0-alpha.1", default-features = false, path = "../../../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../../primitives/runtime" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "../../common" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../../../primitives/runtime" } +pallet-contracts-primitives = { version = "2.0.0-alpha.1", default-features = false, path = "../../common" } [features] default = ["std"] diff --git a/frame/contracts/rpc/runtime-api/src/lib.rs b/frame/contracts/rpc/runtime-api/src/lib.rs index fd83ba6534a4524c8c35c2e290f9138157698094..6fb629b02458129e0bc8ffe13bdf363cd45479e1 100644 --- a/frame/contracts/rpc/runtime-api/src/lib.rs +++ b/frame/contracts/rpc/runtime-api/src/lib.rs @@ -76,7 +76,7 @@ sp_api::decl_runtime_apis! { /// Returns the projected time a given contract will be able to sustain paying its rent. /// - /// The returned projection is relevent for the current block, i.e. it is as if the contract + /// The returned projection is relevant for the current block, i.e. it is as if the contract /// was accessed at the current block. /// /// Returns `Err` if the contract is in a tombstone state or doesn't exist. diff --git a/frame/contracts/rpc/src/lib.rs b/frame/contracts/rpc/src/lib.rs index b0d6037416edb2e180608ce8bb5f853c41b15fd0..52dddb177bbc74538f869419d20beebd1cc2b916 100644 --- a/frame/contracts/rpc/src/lib.rs +++ b/frame/contracts/rpc/src/lib.rs @@ -138,7 +138,7 @@ pub trait ContractsApi { /// Returns the projected time a given contract will be able to sustain paying its rent. /// - /// The returned projection is relevent for the given block, i.e. it is as if the contract was + /// The returned projection is relevant for the given block, i.e. it is as if the contract was /// accessed at the beginning of that block. /// /// Returns `None` if the contract is exempted from rent. diff --git a/frame/contracts/src/account_db.rs b/frame/contracts/src/account_db.rs index 814983c5860f63b09b56ce98d24c0484a03bc3be..374c55c374d7bf26c0603017f3838b9c0a6a0bb7 100644 --- a/frame/contracts/src/account_db.rs +++ b/frame/contracts/src/account_db.rs @@ -26,7 +26,7 @@ use sp_std::collections::btree_map::{BTreeMap, Entry}; use sp_std::prelude::*; use sp_io::hashing::blake2_256; use sp_runtime::traits::{Bounded, Zero}; -use frame_support::traits::{Currency, Get, Imbalance, SignedImbalance, UpdateBalanceOutcome}; +use frame_support::traits::{Currency, Get, Imbalance, SignedImbalance}; use frame_support::{storage::child, StorageMap}; use frame_system; @@ -137,7 +137,7 @@ impl AccountDb for DirectAccountDb { >::get(account).and_then(|i| i.as_alive().map(|i| i.rent_allowance)) } fn contract_exists(&self, account: &T::AccountId) -> bool { - >::exists(account) + >::contains_key(account) } fn get_balance(&self, account: &T::AccountId) -> BalanceOf { T::Currency::free_balance(account) @@ -146,10 +146,12 @@ impl AccountDb for DirectAccountDb { let mut total_imbalance = SignedImbalance::zero(); for (address, changed) in s.into_iter() { if let Some(balance) = changed.balance() { - let (imbalance, outcome) = T::Currency::make_free_balance_be(&address, balance); + let existed = !T::Currency::total_balance(&address).is_zero(); + let imbalance = T::Currency::make_free_balance_be(&address, balance); + let exists = !T::Currency::total_balance(&address).is_zero(); total_imbalance = total_imbalance.merge(imbalance); - if let UpdateBalanceOutcome::AccountKilled = outcome { - // Account killed. This will ultimately lead to calling `OnReapAccount` callback + if existed && !exists { + // Account killed. This will ultimately lead to calling `OnKilledAccount` callback // which will make removal of CodeHashOf and AccountStorage for this account. // In order to avoid writing over the deleted properties we `continue` here. continue; diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index d0033e0bd128a8b003bb33db914f33d129b2bc87..53a6c484fccdbd992a67175909b7b6e90d0665a0 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -554,7 +554,6 @@ where #[derive(Copy, Clone)] pub enum TransferFeeKind { ContractInstantiate, - AccountCreate, Transfer, } @@ -572,7 +571,6 @@ impl Token for TransferFeeToken> { fn calculate_amount(&self, metadata: &Config) -> Gas { let balance_fee = match self.kind { TransferFeeKind::ContractInstantiate => metadata.contract_account_instantiate_fee, - TransferFeeKind::AccountCreate => metadata.account_create_fee, TransferFeeKind::Transfer => return metadata.schedule.transfer_cost, }; approx_gas_for_balance(self.gas_price, balance_fee) @@ -612,28 +610,14 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( use self::TransferCause::*; use self::TransferFeeKind::*; - let to_balance = ctx.overlay.get_balance(dest); - - // `would_create` indicates whether the account will be created if this transfer gets executed. - // This flag is orthogonal to `cause. - // For example, we can instantiate a contract at the address which already has some funds. In this - // `would_create` will be `false`. Another example would be when this function is called from `call`, - // and account with the address `dest` doesn't exist yet `would_create` will be `true`. - let would_create = to_balance.is_zero(); - let token = { let kind: TransferFeeKind = match cause { // If this function is called from `Instantiate` routine, then we always // charge contract account creation fee. Instantiate => ContractInstantiate, - // Otherwise the fee depends on whether we create a new account or transfer - // to an existing one. - Call => if would_create { - TransferFeeKind::AccountCreate - } else { - TransferFeeKind::Transfer - }, + // Otherwise the fee is to transfer to an account. + Call => TransferFeeKind::Transfer, }; TransferFeeToken { kind, @@ -651,7 +635,8 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( Some(b) => b, None => Err("balance too low to send value")?, }; - if would_create && value < ctx.config.existential_deposit { + let to_balance = ctx.overlay.get_balance(dest); + if to_balance.is_zero() && value < ctx.config.existential_deposit { Err("value too low to create account")? } T::Currency::ensure_can_withdraw( @@ -1009,7 +994,7 @@ mod tests { let mut gas_meter = GasMeter::::with_limit(1000, 1); - let result = ctx.instantiate(0, &mut gas_meter, &code, vec![]); + let result = ctx.instantiate(1, &mut gas_meter, &code, vec![]); assert_matches!(result, Ok(_)); let mut toks = gas_meter.tokens().iter(); @@ -1105,7 +1090,7 @@ mod tests { toks, ExecFeeToken::Call, TransferFeeToken { - kind: TransferFeeKind::AccountCreate, + kind: TransferFeeKind::Transfer, gas_price: 1u64 }, ); @@ -1137,7 +1122,7 @@ mod tests { ); }); - // This test sends 50 units of currency as an endownment to a newly + // This test sends 50 units of currency as an endowment to a newly // instantiated contract. ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let mut loader = MockLoader::empty(); @@ -1302,8 +1287,10 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); + ctx.overlay.set_balance(&ALICE, 1); + let result = ctx.instantiate( - 0, + 1, &mut GasMeter::::with_limit(10000, 1), &input_data_ch, vec![1, 2, 3, 4], @@ -1348,6 +1335,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); + ctx.overlay.set_balance(&BOB, 1); ctx.overlay.instantiate_contract(&BOB, recurse_ch).unwrap(); let result = ctx.call( @@ -1661,8 +1649,10 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); + ctx.overlay.set_balance(&ALICE, 1); + let result = ctx.instantiate( - 0, + 1, &mut GasMeter::::with_limit(10000, 1), &rent_allowance_ch, vec![], diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index e0cc7d4bfb4c7f522d266ffda3303a62bee18a16..c8572daaa43d487efd84725b26a6492beaa63794 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -17,7 +17,7 @@ use crate::{GasSpent, Module, Trait, BalanceOf, NegativeImbalanceOf}; use sp_std::convert::TryFrom; use sp_runtime::traits::{ - CheckedMul, Zero, SaturatedConversion, SimpleArithmetic, UniqueSaturatedInto, + CheckedMul, Zero, SaturatedConversion, AtLeast32Bit, UniqueSaturatedInto, }; use frame_support::{ traits::{Currency, ExistenceRequirement, Imbalance, OnUnbalanced, WithdrawReason}, StorageValue, @@ -248,7 +248,7 @@ pub fn refund_unused_gas( /// A little handy utility for converting a value in balance units into approximate value in gas units /// at the given gas price. pub fn approx_gas_for_balance(gas_price: Balance, balance: Balance) -> Gas - where Balance: SimpleArithmetic + where Balance: AtLeast32Bit { (balance / gas_price).saturated_into::() } diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index bd1e91f1a9d66e8278b7cd987d0c07af6d057a59..571ae9700cf949a534fce77a14c165e25f92b49a 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -125,7 +125,7 @@ use frame_support::{ parameter_types, IsSubType, weights::DispatchInfo, }; -use frame_support::traits::{OnReapAccount, OnUnbalanced, Currency, Get, Time, Randomness}; +use frame_support::traits::{OnKilledAccount, OnUnbalanced, Currency, Get, Time, Randomness}; use frame_system::{self as system, ensure_signed, RawOrigin, ensure_root}; use sp_core::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use pallet_contracts_primitives::{RentProjection, ContractAccessError}; @@ -143,7 +143,7 @@ pub trait ComputeDispatchFee { fn compute_dispatch_fee(call: &Call) -> Balance; } -/// Information for managing an acocunt and its sub trie abstraction. +/// Information for managing an account and its sub trie abstraction. /// This is the required info to cache for an account #[derive(Encode, Decode, RuntimeDebug)] pub enum ContractInfo { @@ -401,9 +401,6 @@ pub trait Trait: frame_system::Trait { /// to removal of a contract. type SurchargeReward: Get>; - /// The fee required to create an account. - type CreationFee: Get>; - /// The fee to be paid for making a transaction; the base. type TransactionBaseFee: Get>; @@ -435,8 +432,8 @@ pub trait Trait: frame_system::Trait { /// and the account id that requested the account creation. /// /// Formula: `blake2_256(blake2_256(code) + blake2_256(data) + origin)` -pub struct SimpleAddressDeterminator(PhantomData); -impl ContractAddressFor, T::AccountId> for SimpleAddressDeterminator +pub struct SimpleAddressDeterminer(PhantomData); +impl ContractAddressFor, T::AccountId> for SimpleAddressDeterminer where T::AccountId: UncheckedFrom + AsRef<[u8]> { @@ -497,7 +494,7 @@ decl_module! { /// The minimum amount required to generate a tombstone. const TombstoneDeposit: BalanceOf = T::TombstoneDeposit::get(); - /// Size of a contract at the time of instantiaion. This is a simple way to ensure that + /// Size of a contract at the time of instantiation. This is a simple way to ensure that /// empty contracts eventually gets deleted. const StorageSizeOffset: u32 = T::StorageSizeOffset::get(); @@ -517,9 +514,6 @@ decl_module! { /// to removal of a contract. const SurchargeReward: BalanceOf = T::SurchargeReward::get(); - /// The fee required to create an account. - const CreationFee: BalanceOf = T::CreationFee::get(); - /// The fee to be paid for making a transaction; the base. const TransactionBaseFee: BalanceOf = T::TransactionBaseFee::get(); @@ -947,8 +941,12 @@ decl_storage! { } } -impl OnReapAccount for Module { - fn on_reap_account(who: &T::AccountId) { +// TODO: this should be removed in favour of a self-destruct contract host function allowing the +// contract to delete all storage and the `ContractInfoOf` key and transfer remaining balance to +// some other account. As it stands, it's an economic insecurity on any smart-contract chain. +// https://github.com/paritytech/substrate/issues/4952 +impl OnKilledAccount for Module { + fn on_killed_account(who: &T::AccountId) { if let Some(ContractInfo::Alive(info)) = >::take(who) { child::kill_storage(&info.trie_id, info.child_trie_unique_id()); } @@ -966,7 +964,6 @@ pub struct Config { pub max_depth: u32, pub max_value_size: u32, pub contract_account_instantiate_fee: BalanceOf, - pub account_create_fee: BalanceOf, } impl Config { @@ -978,7 +975,6 @@ impl Config { max_depth: T::MaxDepth::get(), max_value_size: T::MaxValueSize::get(), contract_account_instantiate_fee: T::ContractFee::get(), - account_create_fee: T::CreationFee::get(), } } } diff --git a/frame/contracts/src/rent.rs b/frame/contracts/src/rent.rs index 49beebbf0c20270137e4221ac33757993f432a10..8b6825419c81e570033f695f52cf03c1f20847b6 100644 --- a/frame/contracts/src/rent.rs +++ b/frame/contracts/src/rent.rs @@ -327,7 +327,7 @@ pub fn snitch_contract_should_be_evicted( } /// Returns the projected time a given contract will be able to sustain paying its rent. The -/// returned projection is relevent for the current block, i.e. it is as if the contract was +/// returned projection is relevant for the current block, i.e. it is as if the contract was /// accessed at the beginning of the current block. Returns `None` in case if the contract was /// evicted before or as a result of the rent collection. /// diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 5eb7bce48ab3fc3b27ad036c0e4baaba0bb166c4..e775998a3a55331443d37fad3cad78275e7f0527 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -41,7 +41,7 @@ use std::{cell::RefCell, sync::atomic::{AtomicUsize, Ordering}}; use sp_core::storage::well_known_keys; use frame_system::{self as system, EventRecord, Phase}; -mod contract { +mod contracts { // Re-export contents of the root. This basically // needs to give a name for the current crate. // This hack is required for `impl_outer_event!`. @@ -53,7 +53,9 @@ use pallet_balances as balances; impl_outer_event! { pub enum MetaEvent for Test { - balances, contract, + system, + balances, + contracts, } } impl_outer_origin! { @@ -62,7 +64,7 @@ impl_outer_origin! { impl_outer_dispatch! { pub enum Call for Test where origin: Origin { balances::Balances, - contract::Contract, + contracts::Contracts, } } @@ -83,11 +85,6 @@ impl Get for TransferFee { fn get() -> u64 { TRANSFER_FEE.with(|v| *v.borrow()) } } -pub struct CreationFee; -impl Get for CreationFee { - fn get() -> u64 { INSTANTIATION_FEE.with(|v| *v.borrow()) } -} - pub struct BlockGasLimit; impl Get for BlockGasLimit { fn get() -> u64 { BLOCK_GAS_LIMIT.with(|v| *v.borrow()) } @@ -118,16 +115,16 @@ impl frame_system::Trait for Test { type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = Contracts; } impl pallet_balances::Trait for Test { type Balance = u64; - type OnReapAccount = (System, Contract); - type OnNewAccount = (); type Event = MetaEvent; type DustRemoval = (); - type TransferPayment = (); type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = System; } parameter_types! { pub const MinimumPeriod: u64 = 1; @@ -169,7 +166,6 @@ impl Trait for Test { type RentByteFee = RentByteFee; type RentDepositOffset = RentDepositOffset; type SurchargeReward = SurchargeReward; - type CreationFee = CreationFee; type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type ContractFee = ContractFee; @@ -182,7 +178,7 @@ impl Trait for Test { type Balances = pallet_balances::Module; type Timestamp = pallet_timestamp::Module; -type Contract = Module; +type Contracts = Module; type System = frame_system::Module; type Randomness = pallet_randomness_collective_flip::Module; @@ -235,7 +231,7 @@ pub struct ExtBuilder { impl Default for ExtBuilder { fn default() -> Self { Self { - existential_deposit: 0, + existential_deposit: 1, gas_price: 2, block_gas_limit: 100_000_000, transfer_fee: 0, @@ -304,7 +300,7 @@ fn refunds_unused_gas() { ExtBuilder::default().gas_price(2).build().execute_with(|| { Balances::deposit_creating(&ALICE, 100_000_000); - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, Vec::new())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, Vec::new())); // 2 * 135 - gas price multiplied by the call base fee. assert_eq!(Balances::free_balance(ALICE), 100_000_000 - (2 * 135)); @@ -413,10 +409,10 @@ fn instantiate_and_call_and_deposit_event() { ExtBuilder::default().existential_deposit(100).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Check at the end to get hash on error easily - let creation = Contract::instantiate( + let creation = Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, @@ -427,40 +423,50 @@ fn instantiate_and_call_and_deposit_event() { assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), event: MetaEvent::balances( - pallet_balances::RawEvent::NewAccount(BOB, 100) + pallet_balances::RawEvent::Endowed(BOB, 100) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Transfer(ALICE, BOB, 100)), + event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::ContractExecution(BOB, vec![1, 2, 3, 4])), + event: MetaEvent::contracts(RawEvent::ContractExecution(BOB, vec![1, 2, 3, 4])), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Instantiated(ALICE, BOB)), + event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), topics: vec![], } ]); assert_ok!(creation); - assert!(ContractInfoOf::::exists(BOB)); + assert!(ContractInfoOf::::contains_key(BOB)); }); } @@ -493,24 +499,29 @@ fn dispatch_call() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Let's keep this assert even though it's redundant. If you ever need to update the // wasm source this test will fail and will show you the actual hash. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), topics: vec![], }, ]); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, @@ -518,7 +529,7 @@ fn dispatch_call() { vec![], )); - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, // newly created account 0, @@ -529,52 +540,67 @@ fn dispatch_call() { assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), event: MetaEvent::balances( - pallet_balances::RawEvent::NewAccount(BOB, 100) + pallet_balances::RawEvent::Endowed(BOB, 100) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Transfer(ALICE, BOB, 100)), + event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Instantiated(ALICE, BOB)), + event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), topics: vec![], }, // Dispatching the call. + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(CHARLIE)), + topics: vec![], + }, EventRecord { phase: Phase::ApplyExtrinsic(0), event: MetaEvent::balances( - pallet_balances::RawEvent::NewAccount(CHARLIE, 50) + pallet_balances::RawEvent::Endowed(CHARLIE, 50) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), event: MetaEvent::balances( - pallet_balances::RawEvent::Transfer(BOB, CHARLIE, 50, 0) + pallet_balances::RawEvent::Transfer(BOB, CHARLIE, 50) ), topics: vec![], }, - // Event emited as a result of dispatch. + // Event emitted as a result of dispatch. EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Dispatched(BOB, true)), + event: MetaEvent::contracts(RawEvent::Dispatched(BOB, true)), topics: vec![], } ]); @@ -611,24 +637,29 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Let's keep this assert even though it's redundant. If you ever need to update the // wasm source this test will fail and will show you the actual hash. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), topics: vec![], }, ]); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, @@ -639,7 +670,7 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { // Call the newly instantiated contract. The contract is expected to dispatch a call // and then trap. assert_err!( - Contract::call( + Contracts::call( Origin::signed(ALICE), BOB, // newly created account 0, @@ -651,29 +682,39 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), event: MetaEvent::balances( - pallet_balances::RawEvent::NewAccount(BOB, 100) + pallet_balances::RawEvent::Endowed(BOB, 100) ), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Transfer(ALICE, BOB, 100)), + event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Instantiated(ALICE, BOB)), + event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), topics: vec![], }, // ABSENCE of events which would be caused by dispatched Balances::transfer call @@ -681,6 +722,51 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { }); } +const CODE_RUN_OUT_OF_GAS: &str = r#" +(module + (func (export "call") + (loop $inf (br $inf)) ;; just run out of gas + (unreachable) + ) + (func (export "deploy")) +) +"#; + +#[test] +fn run_out_of_gas() { + let (wasm, code_hash) = compile_module::(CODE_RUN_OUT_OF_GAS).unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + Balances::deposit_creating(&ALICE, 1_000_000); + + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100, + 100_000, + code_hash.into(), + vec![], + )); + + // Call the contract with a fixed gas limit. It must run out of gas because it just + // loops forever. + assert_err!( + Contracts::call( + Origin::signed(ALICE), + BOB, // newly created account + 0, + 1000, + vec![], + ), + "ran out of gas during contract execution" + ); + }); +} + const CODE_SET_RENT: &str = r#" (module (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) @@ -810,19 +896,24 @@ fn test_set_rent_code_and_hash() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // If you ever need to update the wasm source this test will fail // and will show you the actual hash. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(code_hash.into())), + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), topics: vec![], }, ]); @@ -837,8 +928,8 @@ fn storage_size() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, 100_000, code_hash.into(), @@ -847,11 +938,11 @@ fn storage_size() { let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::set_storage_4_byte())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::set_storage_4_byte())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4 + 4); - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::remove_storage_4_byte())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::remove_storage_4_byte())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); }); @@ -874,8 +965,8 @@ fn deduct_blocks() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, 100_000, code_hash.into(), @@ -890,7 +981,7 @@ fn deduct_blocks() { initialize_block(5); // Trigger rent through call - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); // Check result let rent = (8 + 4 - 3) // storage size = size_offset + deploy_set_storage - deposit_offset @@ -905,7 +996,7 @@ fn deduct_blocks() { initialize_block(12); // Trigger rent through call - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); // Check result let rent_2 = (8 + 4 - 2) // storage size = size_offset + deploy_set_storage - deposit_offset @@ -917,7 +1008,7 @@ fn deduct_blocks() { assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); // Second call on same block should have no effect on rent - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); @@ -930,46 +1021,46 @@ fn deduct_blocks() { fn call_contract_removals() { removals(|| { // Call on already-removed account might fail, and this is fine. - Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()); + Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()); true }); } #[test] fn inherent_claim_surcharge_contract_removals() { - removals(|| Contract::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok()); + removals(|| Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok()); } #[test] fn signed_claim_surcharge_contract_removals() { - removals(|| Contract::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok()); + removals(|| Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok()); } #[test] fn claim_surcharge_malus() { // Test surcharge malus for inherent - claim_surcharge(4, || Contract::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(3, || Contract::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(2, || Contract::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(1, || Contract::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), false); + claim_surcharge(4, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(3, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(2, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(1, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), false); // Test surcharge malus for signed - claim_surcharge(4, || Contract::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), true); - claim_surcharge(3, || Contract::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); - claim_surcharge(2, || Contract::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); - claim_surcharge(1, || Contract::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); + claim_surcharge(4, || Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), true); + claim_surcharge(3, || Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); + claim_surcharge(2, || Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); + claim_surcharge(1, || Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), false); } /// Claim surcharge with the given trigger_call at the given blocks. -/// if removes is true then assert that the contract is a tombstonedead +/// If `removes` is true then assert that the contract is a tombstone. fn claim_surcharge(blocks: u64, trigger_call: impl Fn() -> bool, removes: bool) { let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, code_hash.into(), @@ -1001,8 +1092,8 @@ fn removals(trigger_call: impl Fn() -> bool) { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, code_hash.into(), @@ -1037,8 +1128,8 @@ fn removals(trigger_call: impl Fn() -> bool) { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 1_000, 100_000, code_hash.into(), @@ -1072,8 +1163,8 @@ fn removals(trigger_call: impl Fn() -> bool) { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 50+Balances::minimum_balance(), 100_000, code_hash.into(), @@ -1086,7 +1177,7 @@ fn removals(trigger_call: impl Fn() -> bool) { assert_eq!(Balances::free_balance(BOB), 50 + Balances::minimum_balance()); // Transfer funds - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::transfer())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::transfer())); assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); @@ -1116,8 +1207,8 @@ fn call_removed_contract() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, code_hash.into(), @@ -1125,28 +1216,28 @@ fn call_removed_contract() { )); // Calling contract should succeed. - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); // Advance blocks initialize_block(10); // Calling contract should remove contract and fail. assert_err!( - Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), "contract has been evicted" ); // Calling a contract that is about to evict shall emit an event. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Evicted(BOB, true)), + event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), topics: vec![], }, ]); // Subsequent contract calls should also fail. assert_err!( - Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), "contract has been evicted" ); }) @@ -1209,8 +1300,8 @@ fn default_rent_allowance_on_instantiate() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, 100_000, @@ -1226,7 +1317,7 @@ fn default_rent_allowance_on_instantiate() { initialize_block(5); // Trigger rent through call - assert_ok!(Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); // Check contract is still alive let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive(); @@ -1322,32 +1413,37 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, restoration_wasm)); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, set_rent_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, restoration_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, set_rent_wasm)); // If you ever need to update the wasm source this test will fail // and will show you the actual hash. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(1, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(restoration_code_hash.into())), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::CodeStored(set_rent_code_hash.into())), + event: MetaEvent::contracts(RawEvent::CodeStored(restoration_code_hash.into())), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::contracts(RawEvent::CodeStored(set_rent_code_hash.into())), topics: vec![], }, ]); // Create an account with address `BOB` with code `CODE_SET_RENT`. // The input parameter sets the rent allowance to 0. - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, 100_000, @@ -1361,7 +1457,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_eq!(bob_contract.rent_allowance, 0); if test_different_storage { - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, 100_000, call::set_storage_4_byte()) @@ -1377,14 +1473,14 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: // Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0 // we expect that it will get removed leaving tombstone. assert_err!( - Contract::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), "contract has been evicted" ); assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract( + event: MetaEvent::contracts( RawEvent::Evicted(BOB.clone(), true) ), topics: vec![], @@ -1396,7 +1492,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: /// Note that we can't use `ALICE` for creating `DJANGO` so we create yet another /// account `CHARLIE` and create `DJANGO` with it. Balances::deposit_creating(&CHARLIE, 1_000_000); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(CHARLIE), 30_000, 100_000, @@ -1415,7 +1511,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: // Perform a call to `DJANGO`. This should either perform restoration successfully or // fail depending on the test parameters. - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), DJANGO, 0, @@ -1437,7 +1533,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract( + event: MetaEvent::contracts( RawEvent::Restored(DJANGO, BOB, bob_code_hash, 50, false) ), topics: vec![], @@ -1448,32 +1544,42 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Evicted(BOB, true)), + event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(CHARLIE)), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(CHARLIE, 1_000_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(CHARLIE, 1_000_000)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(DJANGO)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(pallet_balances::RawEvent::NewAccount(DJANGO, 30_000)), + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(DJANGO, 30_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Transfer(CHARLIE, DJANGO, 30_000)), + event: MetaEvent::contracts(RawEvent::Transfer(CHARLIE, DJANGO, 30_000)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Instantiated(CHARLIE, DJANGO)), + event: MetaEvent::contracts(RawEvent::Instantiated(CHARLIE, DJANGO)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract(RawEvent::Restored( + event: MetaEvent::contracts(RawEvent::Restored( DJANGO, BOB, bob_code_hash, @@ -1500,12 +1606,12 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::balances(balances::RawEvent::ReapedAccount(DJANGO, 0)), + event: MetaEvent::system(system::RawEvent::KilledAccount(DJANGO)), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::contract( + event: MetaEvent::contracts( RawEvent::Restored(DJANGO, BOB, bob_contract.code_hash, 50, true) ), topics: vec![], @@ -1586,8 +1692,8 @@ fn storage_max_value_limit() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, 100_000, @@ -1600,7 +1706,7 @@ fn storage_max_value_limit() { assert_eq!(bob_contract.rent_allowance, >::max_value()); // Call contract with allowed storage value. - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -1610,7 +1716,7 @@ fn storage_max_value_limit() { // Call contract with too large a storage value. assert_err!( - Contract::call( + Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -1950,10 +2056,10 @@ fn deploy_and_call_other_contract() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, 100_000, @@ -1963,7 +2069,7 @@ fn deploy_and_call_other_contract() { // Call BOB contract, which attempts to instantiate and call the callee contract and // makes various assertions on the results from those calls. - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -2013,11 +2119,11 @@ const CODE_SELF_DESTRUCT: &str = r#" ;; Read own address into memory. (call $ext_scratch_read (i32.const 16) ;; Pointer to write address to - (i32.const 0) ;; Offset into scrach buffer + (i32.const 0) ;; Offset into scratch buffer (i32.const 8) ;; Length of encoded address ) - ;; Recursively call self with empty imput data. + ;; Recursively call self with empty input data. (call $assert (i32.eq (call $ext_call @@ -2049,7 +2155,7 @@ const CODE_SELF_DESTRUCT: &str = r#" ;; Read balance into memory. (call $ext_scratch_read (i32.const 8) ;; Pointer to write balance to - (i32.const 0) ;; Offset into scrach buffer + (i32.const 0) ;; Offset into scratch buffer (i32.const 8) ;; Length of encoded balance ) @@ -2077,10 +2183,10 @@ fn self_destruct_by_draining_balance() { let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCT).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Instantiate the BOB contract. - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, 100_000, @@ -2095,7 +2201,7 @@ fn self_destruct_by_draining_balance() { ); // Call BOB with no input data, forcing it to self-destruct. - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -2113,10 +2219,10 @@ fn cannot_self_destruct_while_live() { let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCT).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Instantiate the BOB contract. - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, 100_000, @@ -2133,7 +2239,7 @@ fn cannot_self_destruct_while_live() { // Call BOB with input data, forcing it make a recursive call to itself to // self-destruct, resulting in a trap. assert_err!( - Contract::call( + Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -2313,12 +2419,12 @@ fn destroy_contract_and_transfer_funds() { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); // This deploys the BOB contract, which in turn deploys the CHARLIE contract during // construction. - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 200_000, 100_000, @@ -2333,7 +2439,7 @@ fn destroy_contract_and_transfer_funds() { ); // Call BOB, which calls CHARLIE, forcing CHARLIE to self-destruct. - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, @@ -2378,7 +2484,7 @@ const CODE_SELF_DESTRUCTING_CONSTRUCTOR: &str = r#" ;; Read balance into memory. (call $ext_scratch_read (i32.const 8) ;; Pointer to write balance to - (i32.const 0) ;; Offset into scrach buffer + (i32.const 0) ;; Offset into scratch buffer (i32.const 8) ;; Length of encoded balance ) @@ -2408,12 +2514,12 @@ fn cannot_self_destruct_in_constructor() { let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCTING_CONSTRUCTOR).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); // Fail to instantiate the BOB contract since its final balance is below existential // deposit. assert_err!( - Contract::instantiate( + Contracts::instantiate( Origin::signed(ALICE), 100_000, 100_000, @@ -2529,15 +2635,15 @@ fn get_runtime_storage() { 0x14144020u32.to_le_bytes().to_vec().as_ref() ); - assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contract::instantiate( + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, 100_000, code_hash.into(), vec![], )); - assert_ok!(Contract::call( + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index 66ae8a4996ca49ff39b3f23b586658493659309e..a84556d884ce97a3a2c16fec8b1051a6d72cfe35 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -39,6 +39,8 @@ const TRAP_RETURN_CODE: u32 = 0x0100; enum SpecialTrap { /// Signals that trap was generated in response to call `ext_return` host function. Return(Vec), + /// Signals that trap was generated because the contract exhausted its gas limit. + OutOfGas, } /// Can only be used for one call. @@ -74,9 +76,21 @@ pub(crate) fn to_execution_result( runtime: Runtime, sandbox_result: Result, ) -> ExecResult { - // Special case. The trap was the result of the execution `return` host function. - if let Some(SpecialTrap::Return(data)) = runtime.special_trap { - return Ok(ExecReturnValue { status: STATUS_SUCCESS, data }); + match runtime.special_trap { + // The trap was the result of the execution `return` host function. + Some(SpecialTrap::Return(data)) => { + return Ok(ExecReturnValue { + status: STATUS_SUCCESS, + data, + }) + } + Some(SpecialTrap::OutOfGas) => { + return Err(ExecError { + reason: "ran out of gas during contract execution".into(), + buffer: runtime.scratch_buf, + }) + } + _ => (), } // Check the exact type of the error. @@ -179,11 +193,15 @@ impl Token for RuntimeToken { fn charge_gas>( gas_meter: &mut GasMeter, metadata: &Tok::Metadata, + special_trap: &mut Option, token: Tok, ) -> Result<(), sp_sandbox::HostError> { match gas_meter.charge(metadata, token) { GasMeterResult::Proceed => Ok(()), - GasMeterResult::OutOfGas => Err(sp_sandbox::HostError), + GasMeterResult::OutOfGas => { + *special_trap = Some(SpecialTrap::OutOfGas); + Err(sp_sandbox::HostError) + }, } } @@ -200,7 +218,12 @@ fn read_sandbox_memory( ptr: u32, len: u32, ) -> Result, sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(len), + )?; let mut buf = vec![0u8; len as usize]; ctx.memory.get(ptr, buf.as_mut_slice()).map_err(|_| sp_sandbox::HostError)?; @@ -220,7 +243,12 @@ fn read_sandbox_memory_into_scratch( ptr: u32, len: u32, ) -> Result<(), sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(len), + )?; ctx.scratch_buf.resize(len as usize, 0); ctx.memory.get(ptr, ctx.scratch_buf.as_mut_slice()).map_err(|_| sp_sandbox::HostError)?; @@ -240,7 +268,12 @@ fn read_sandbox_memory_into_buf( ptr: u32, buf: &mut [u8], ) -> Result<(), sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(buf.len() as u32))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(buf.len() as u32), + )?; ctx.memory.get(ptr, buf).map_err(Into::into) } @@ -273,12 +306,18 @@ fn read_sandbox_memory_as( /// - designated area is not within the bounds of the sandbox memory. fn write_sandbox_memory( schedule: &Schedule, + special_trap: &mut Option, gas_meter: &mut GasMeter, memory: &sp_sandbox::Memory, ptr: u32, buf: &[u8], ) -> Result<(), sp_sandbox::HostError> { - charge_gas(gas_meter, schedule, RuntimeToken::WriteMemory(buf.len() as u32))?; + charge_gas( + gas_meter, + schedule, + special_trap, + RuntimeToken::WriteMemory(buf.len() as u32), + )?; memory.set(ptr, buf)?; @@ -300,7 +339,12 @@ define_env!(Env, , // // - amount: How much gas is used. gas(ctx, amount: u32) => { - charge_gas(&mut ctx.gas_meter, ctx.schedule, RuntimeToken::Explicit(amount))?; + charge_gas( + &mut ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::Explicit(amount) + )?; Ok(()) }, @@ -520,7 +564,12 @@ define_env!(Env, , // // This is the only way to return a data buffer to the caller. ext_return(ctx, data_ptr: u32, data_len: u32) => { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReturnData(data_len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReturnData(data_len) + )?; read_sandbox_memory_into_scratch(ctx, data_ptr, data_len)?; let output_buf = mem::replace(&mut ctx.scratch_buf, Vec::new()); @@ -652,7 +701,12 @@ define_env!(Env, , let balance_fee = <::T as Trait>::ComputeDispatchFee::compute_dispatch_fee(&call); approx_gas_for_balance(ctx.gas_meter.gas_price(), balance_fee) }; - charge_gas(&mut ctx.gas_meter, ctx.schedule, RuntimeToken::ComputedDispatchFee(fee))?; + charge_gas( + &mut ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ComputedDispatchFee(fee) + )?; ctx.ext.note_dispatch_call(call); @@ -662,10 +716,10 @@ define_env!(Env, , // Record a request to restore the caller contract to the specified contract. // // At the finalization stage, i.e. when all changes from the extrinsic that invoked this - // contract are commited, this function will compute a tombstone hash from the caller's + // contract are committed, this function will compute a tombstone hash from the caller's // storage and the given code hash and if the hash matches the hash found in the tombstone at // the specified address - kill the caller contract and restore the destination contract and set - // the specified `rent_allowance`. All caller's funds are transfered to the destination. + // the specified `rent_allowance`. All caller's funds are transferred to the destination. // // This function doesn't perform restoration right away but defers it to the end of the // transaction. If there is no tombstone in the destination address or if the hashes don't match @@ -756,6 +810,7 @@ define_env!(Env, , // Finally, perform the write. write_sandbox_memory( ctx.schedule, + &mut ctx.special_trap, ctx.gas_meter, &ctx.memory, dest_ptr, @@ -803,6 +858,7 @@ define_env!(Env, , charge_gas( ctx.gas_meter, ctx.schedule, + &mut ctx.special_trap, RuntimeToken::DepositEvent(topics.len() as u32, data_len) )?; ctx.ext.deposit_event(topics, event_data); diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index 2428d2c1544dcf7309294f9238b2031cb8cd3210..512a4676cb174b6d63149a52535a09f5bde22991 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "pallet-democracy" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0", path = "../balances" } -sp-storage = { version = "2.0.0", path = "../../primitives/storage" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../balances" } +sp-storage = { version = "2.0.0-alpha.1", path = "../../primitives/storage" } hex-literal = "0.2.1" [features] diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index da0cccb1d0e6bda815298332da43f9ce19b46de8..6755fabd2039f1eb67549382769aa944627787b7 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -30,7 +30,7 @@ use frame_support::{ weights::SimpleDispatchInfo, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, - OnReapAccount, OnUnbalanced + OnUnbalanced, BalanceStatus } }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -260,6 +260,24 @@ impl ReferendumInfo } } +/// State of a proxy voting account. +#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] +pub enum ProxyState { + /// Account is open to becoming a proxy but is not yet assigned. + Open(AccountId), + /// Account is actively being a proxy. + Active(AccountId), +} + +impl ProxyState { + fn as_active(self) -> Option { + match self { + ProxyState::Active(a) => Some(a), + ProxyState::Open(_) => None, + } + } +} + decl_storage! { trait Store for Module as Democracy { /// The number of (public) proposals that have been made so far. @@ -294,12 +312,12 @@ decl_storage! { /// Get the vote in a given referendum of a particular voter. The result is meaningful only /// if `voters_for` includes the voter when called with the referendum (you'll get the /// default `Vote` value otherwise). If you don't want to check `voters_for`, then you can - /// also check for simple existence with `VoteOf::exists` first. + /// also check for simple existence with `VoteOf::contains_key` first. pub VoteOf get(fn vote_of): map hasher(blake2_256) (ReferendumIndex, T::AccountId) => Vote; /// Who is able to vote for whom. Value is the fund-holding account, key is the /// vote-transaction-sending account. - pub Proxy get(fn proxy): map hasher(blake2_256) T::AccountId => Option; + pub Proxy get(fn proxy): map hasher(blake2_256) T::AccountId => Option>; /// Get the account (and lock periods) to which another account is delegating vote. pub Delegations get(fn delegations): @@ -423,6 +441,12 @@ decl_error! { NotLocked, /// The lock on the account to be unlocked has not yet expired. NotExpired, + /// A proxy-pairing was attempted to an account that was not open. + NotOpen, + /// A proxy-pairing was attempted to an account that was open to another account. + WrongOpen, + /// A proxy-de-pairing was attempted to an account that was not active. + NotActive } } @@ -525,8 +549,9 @@ decl_module! { #[compact] ref_index: ReferendumIndex, vote: Vote ) -> DispatchResult { - let who = Self::proxy(ensure_signed(origin)?).ok_or(Error::::NotProxy)?; - Self::do_vote(who, ref_index, vote) + let who = ensure_signed(origin)?; + let voter = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; + Self::do_vote(voter, ref_index, vote) } /// Schedule an emergency cancellation of a referendum. Cannot happen twice to the same @@ -537,7 +562,7 @@ decl_module! { let info = Self::referendum_info(ref_index).ok_or(Error::::BadIndex)?; let h = info.proposal_hash; - ensure!(!>::exists(h), Error::::AlreadyCanceled); + ensure!(!>::contains_key(h), Error::::AlreadyCanceled); >::insert(h, true); Self::clear_referendum(ref_index); @@ -659,42 +684,65 @@ decl_module! { } } - /// Specify a proxy. Called by the stash. + /// Specify a proxy that is already open to us. Called by the stash. + /// + /// NOTE: Used to be called `set_proxy`. /// /// # /// - One extra DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn set_proxy(origin, proxy: T::AccountId) { + fn activate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; - ensure!(!>::exists(&proxy), Error::::AlreadyProxy); - >::insert(proxy, who) + Proxy::::try_mutate(&proxy, |a| match a.take() { + None => Err(Error::::NotOpen), + Some(ProxyState::Active(_)) => Err(Error::::AlreadyProxy), + Some(ProxyState::Open(x)) if &x == &who => { + *a = Some(ProxyState::Active(who)); + Ok(()) + } + Some(ProxyState::Open(_)) => Err(Error::::WrongOpen), + })?; } /// Clear the proxy. Called by the proxy. /// + /// NOTE: Used to be called `resign_proxy`. + /// /// # /// - One DB clear. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn resign_proxy(origin) { + fn close_proxy(origin) { let who = ensure_signed(origin)?; - >::remove(who); + Proxy::::mutate(&who, |a| { + if a.is_some() { + system::Module::::dec_ref(&who); + } + *a = None; + }); } - /// Clear the proxy. Called by the stash. + /// Deactivate the proxy, but leave open to this account. Called by the stash. + /// + /// The proxy must already be active. + /// + /// NOTE: Used to be called `remove_proxy`. /// /// # /// - One DB clear. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn remove_proxy(origin, proxy: T::AccountId) { + fn deactivate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; - ensure!( - &Self::proxy(&proxy).ok_or(Error::::NotProxy)? == &who, - Error::::WrongProxy, - ); - >::remove(proxy); + Proxy::::try_mutate(&proxy, |a| match a.take() { + None | Some(ProxyState::Open(_)) => Err(Error::::NotActive), + Some(ProxyState::Active(x)) if &x == &who => { + *a = Some(ProxyState::Open(who)); + Ok(()) + } + Some(ProxyState::Active(_)) => Err(Error::::WrongProxy), + })?; } /// Delegate vote. @@ -725,7 +773,7 @@ decl_module! { #[weight = SimpleDispatchInfo::FixedNormal(500_000)] fn undelegate(origin) { let who = ensure_signed(origin)?; - ensure!(>::exists(&who), Error::::NotDelegated); + ensure!(>::contains_key(&who), Error::::NotDelegated); let (_, conviction) = >::take(&who); // Indefinite lock is reduced to the maximum voting lock that could be possible. let now = >::block_number(); @@ -754,7 +802,7 @@ decl_module! { fn note_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - ensure!(!>::exists(&proposal_hash), Error::::DuplicatePreimage); + ensure!(!>::contains_key(&proposal_hash), Error::::DuplicatePreimage); let deposit = >::from(encoded_proposal.len() as u32) .saturating_mul(T::PreimageByteDeposit::get()); @@ -772,7 +820,7 @@ decl_module! { fn note_imminent_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - ensure!(!>::exists(&proposal_hash), Error::::DuplicatePreimage); + ensure!(!>::contains_key(&proposal_hash), Error::::DuplicatePreimage); let queue = >::get(); ensure!(queue.iter().any(|item| &item.1 == &proposal_hash), Error::::NotImminent); @@ -802,7 +850,7 @@ decl_module! { let queue = >::get(); ensure!(!queue.iter().any(|item| &item.1 == &proposal_hash), Error::::Imminent); - let _ = T::Currency::repatriate_reserved(&old, &who, deposit); + let _ = T::Currency::repatriate_reserved(&old, &who, deposit, BalanceStatus::Free); >::remove(&proposal_hash); Self::deposit_event(RawEvent::PreimageReaped(proposal_hash, old, deposit, who)); } @@ -818,6 +866,30 @@ decl_module! { Locks::::remove(&target); Self::deposit_event(RawEvent::Unlocked(target)); } + + /// Become a proxy. + /// + /// This must be called prior to a later `activate_proxy`. + /// + /// Origin must be a Signed. + /// + /// - `target`: The account whose votes will later be proxied. + /// + /// `close_proxy` must be called before the account can be destroyed. + /// + /// # + /// - One extra DB entry. + /// # + #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + fn open_proxy(origin, target: T::AccountId) { + let who = ensure_signed(origin)?; + Proxy::::mutate(&who, |a| { + if a.is_none() { + system::Module::::inc_ref(&who); + } + *a = Some(ProxyState::Open(target)); + }); + } } } @@ -832,7 +904,7 @@ impl Module { /// Return true if `ref_index` is an on-going referendum. pub fn is_active_referendum(ref_index: ReferendumIndex) -> bool { - >::exists(ref_index) + >::contains_key(ref_index) } /// Get all referenda currently active. @@ -913,7 +985,7 @@ impl Module { if recursion_limit == 0 { return (Zero::zero(), Zero::zero()); } >::enumerate() .filter(|(delegator, (delegate, _))| - *delegate == to && !>::exists(&(ref_index, delegator.clone())) + *delegate == to && !>::contains_key(&(ref_index, delegator.clone())) ).fold( (Zero::zero(), Zero::zero()), |(votes_acc, turnout_acc), (delegator, (_delegate, max_conviction))| { @@ -935,7 +1007,12 @@ impl Module { #[cfg(feature = "std")] pub fn force_proxy(stash: T::AccountId, proxy: T::AccountId) { - >::insert(proxy, stash) + Proxy::::mutate(&proxy, |o| { + if o.is_none() { + system::Module::::inc_ref(&proxy); + } + *o = Some(ProxyState::Active(stash)) + }) } /// Start a referendum. @@ -963,7 +1040,7 @@ impl Module { /// Actually enact a vote, if legit. fn do_vote(who: T::AccountId, ref_index: ReferendumIndex, vote: Vote) -> DispatchResult { ensure!(Self::is_active_referendum(ref_index), Error::::ReferendumInvalid); - if !>::exists((ref_index, &who)) { + if !>::contains_key((ref_index, &who)) { >::append_or_insert(ref_index, &[&who][..]); } >::insert((ref_index, &who), vote); @@ -1162,12 +1239,6 @@ impl Module { } } -impl OnReapAccount for Module { - fn on_reap_account(who: &T::AccountId) { - >::remove(who) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1227,20 +1298,19 @@ mod tests { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; - pub const CreationFee: u64 = 0; + pub const ExistentialDeposit: u64 = 1; } impl pallet_balances::Trait for Test { type Balance = u64; - type OnReapAccount = System; - type OnNewAccount = (); type Event = (); - type TransferPayment = (); type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = System; } parameter_types! { pub const LaunchPeriod: u64 = 2; @@ -1969,29 +2039,45 @@ mod tests { fn proxy_should_work() { new_test_ext().execute_with(|| { assert_eq!(Democracy::proxy(10), None); - assert_ok!(Democracy::set_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), Some(1)); + assert!(System::allow_death(&10)); + + assert_noop!(Democracy::activate_proxy(Origin::signed(1), 10), Error::::NotOpen); + + assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); + assert!(!System::allow_death(&10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); + + assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::WrongOpen); + assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); // Can't set when already set. - assert_noop!(Democracy::set_proxy(Origin::signed(2), 10), Error::::AlreadyProxy); + assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::AlreadyProxy); // But this works because 11 isn't proxying. - assert_ok!(Democracy::set_proxy(Origin::signed(2), 11)); - assert_eq!(Democracy::proxy(10), Some(1)); - assert_eq!(Democracy::proxy(11), Some(2)); + assert_ok!(Democracy::open_proxy(Origin::signed(11), 2)); + assert_ok!(Democracy::activate_proxy(Origin::signed(2), 11)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); + assert_eq!(Democracy::proxy(11), Some(ProxyState::Active(2))); // 2 cannot fire 1's proxy: - assert_noop!(Democracy::remove_proxy(Origin::signed(2), 10), Error::::WrongProxy); + assert_noop!(Democracy::deactivate_proxy(Origin::signed(2), 10), Error::::WrongProxy); - // 1 fires his proxy: - assert_ok!(Democracy::remove_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), None); - assert_eq!(Democracy::proxy(11), Some(2)); + // 1 deactivates their proxy: + assert_ok!(Democracy::deactivate_proxy(Origin::signed(1), 10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); + // but the proxy account cannot be killed until the proxy is closed. + assert!(!System::allow_death(&10)); - // 11 resigns: - assert_ok!(Democracy::resign_proxy(Origin::signed(11))); + // and then 10 closes it completely: + assert_ok!(Democracy::close_proxy(Origin::signed(10))); assert_eq!(Democracy::proxy(10), None); + assert!(System::allow_death(&10)); + + // 11 just closes without 2's "permission". + assert_ok!(Democracy::close_proxy(Origin::signed(11))); assert_eq!(Democracy::proxy(11), None); + assert!(System::allow_death(&11)); }); } @@ -2003,7 +2089,8 @@ mod tests { fast_forward_to(2); let r = 0; - assert_ok!(Democracy::set_proxy(Origin::signed(1), 10)); + assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); + assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); assert_ok!(Democracy::proxy_vote(Origin::signed(10), r, AYE)); assert_eq!(Democracy::voters_for(r), vec![1]); @@ -2067,7 +2154,7 @@ mod tests { } #[test] - /// If transactor already voted, delegated vote is overwriten. + /// If transactor already voted, delegated vote is overwritten. fn single_proposal_should_work_with_vote_and_delegation() { new_test_ext().execute_with(|| { System::set_block_number(0); @@ -2122,7 +2209,7 @@ mod tests { } #[test] - /// If transactor voted, delegated vote is overwriten. + /// If transactor voted, delegated vote is overwritten. fn single_proposal_should_work_with_delegation_and_vote() { new_test_ext().execute_with(|| { System::set_block_number(0); diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index f3cfc800f01c015738304e2281ba6180ffa4b339..882185bdcea93f22d80cb632e48fabb95cc7c3e7 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -1,24 +1,26 @@ [package] name = "pallet-elections-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-phragmen = { version = "2.0.0", default-features = false, path = "../../primitives/phragmen" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-phragmen = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/phragmen" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../primitives/io" } +sp-io = { version = "2.0.0-alpha.1", path = "../../primitives/io" } hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -substrate-test-utils = { version = "2.0.0", path = "../../test-utils" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../balances" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } +substrate-test-utils = { version = "2.0.0-alpha.1", path = "../../test-utils" } serde = { version = "1.0.101" } [features] diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 33d4c46e9125e7a42b01a475fbdcd2fdfe91a7d8..a9474ae844141f178509b847b5ada3280959338f 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -83,12 +83,15 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::prelude::*; -use sp_runtime::{print, DispatchResult, DispatchError, traits::{Zero, StaticLookup, Convert}}; +use sp_runtime::{ + print, DispatchResult, DispatchError, Perbill, + traits::{Zero, StaticLookup, Convert}, +}; use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, weights::SimpleDispatchInfo, traits::{ Currency, Get, LockableCurrency, LockIdentifier, ReservableCurrency, WithdrawReasons, - ChangeMembers, OnUnbalanced, WithdrawReason, Contains + ChangeMembers, OnUnbalanced, WithdrawReason, Contains, BalanceStatus } }; use sp_phragmen::ExtendedBalance; @@ -311,7 +314,7 @@ decl_module! { let valid = Self::is_defunct_voter(&target); if valid { // reporter will get the voting bond of the target - T::Currency::repatriate_reserved(&target, &reporter, T::VotingBond::get())?; + T::Currency::repatriate_reserved(&target, &reporter, T::VotingBond::get(), BalanceStatus::Free)?; // remove the target. They are defunct. Self::do_remove_voter(&target, false); } else { @@ -520,7 +523,7 @@ impl Module { /// /// State: O(1). fn is_voter(who: &T::AccountId) -> bool { - >::exists(who) + >::contains_key(who) } /// Check if `who` is currently an active member. @@ -637,7 +640,7 @@ impl Module { let voters_and_votes = >::enumerate() .map(|(v, i)| (v, i)) .collect::)>>(); - let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote>( + let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote, Perbill>( num_to_elect, 0, candidates, @@ -664,7 +667,7 @@ impl Module { .filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } ) .collect::>(); - let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote>( + let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>( &new_set, &phragmen_result.assignments, Self::locked_stake_of, @@ -811,23 +814,22 @@ mod tests { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); } parameter_types! { pub const ExistentialDeposit: u64 = 1; - pub const CreationFee: u64 = 0; - } +} impl pallet_balances::Trait for Test { type Balance = u64; - type OnNewAccount = (); - type OnReapAccount = System; type Event = Event; - type TransferPayment = (); type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; - } + type AccountStore = frame_system::Module; +} parameter_types! { pub const CandidacyBond: u64 = 3; @@ -934,7 +936,7 @@ mod tests { NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Module, Call, Event}, + System: system::{Module, Call, Event}, Balances: pallet_balances::{Module, Call, Event, Config}, Elections: elections::{Module, Call, Event}, } @@ -1424,7 +1426,7 @@ mod tests { assert_ok!(Elections::report_defunct_voter(Origin::signed(5), 3)); assert_eq!( - System::events()[1].event, + System::events()[7].event, Event::elections(RawEvent::VoterReported(3, 5, true)) ); @@ -1453,7 +1455,7 @@ mod tests { assert_ok!(Elections::report_defunct_voter(Origin::signed(5), 4)); assert_eq!( - System::events()[1].event, + System::events()[7].event, Event::elections(RawEvent::VoterReported(4, 5, false)) ); @@ -1864,7 +1866,7 @@ mod tests { assert_eq!(balances(&5), (45, 2)); assert_eq!( - System::events()[0].event, + System::events()[6].event, Event::elections(RawEvent::NewTerm(vec![(4, 40), (5, 50)])), ); }) diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml index b7c98a65e15b45aa0a52113319102497236b58b5..38eec0dc244c76ba80b3ebcefdfa7a648fe94bba 100644 --- a/frame/elections/Cargo.toml +++ b/frame/elections/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "pallet-elections" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } +pallet-balances = { version = "2.0.0-alpha.1", path = "../balances" } [features] default = ["std"] diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs index 899fcfac8a6f329db76242b4dd28caa28d5f0ca4..95d1858476287aa2c809e3d569c1e56b36638c61 100644 --- a/frame/elections/src/lib.rs +++ b/frame/elections/src/lib.rs @@ -32,7 +32,7 @@ use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, weights::SimpleDispatchInfo, traits::{ - Currency, ExistenceRequirement, Get, LockableCurrency, LockIdentifier, + Currency, ExistenceRequirement, Get, LockableCurrency, LockIdentifier, BalanceStatus, OnUnbalanced, ReservableCurrency, WithdrawReason, WithdrawReasons, ChangeMembers } }; @@ -81,7 +81,7 @@ mod tests; // for B blocks following, there's a counting period whereby each of the candidates that believe // they fall in the top K+C voted can present themselves. they get the total stake -// recorded (based on the snapshot); an ordered list is maintained (the leaderboard). Noone may +// recorded (based on the snapshot); an ordered list is maintained (the leaderboard). No one may // present themselves that, if elected, would result in being included twice in the collective // (important since existing members will have their approval votes as it may be that they // don't get removed), nor if existing presenters would mean they're not in the top K+C. @@ -501,7 +501,7 @@ decl_module! { if valid { // This only fails if `reporter` doesn't exist, which it clearly must do since its // the origin. Still, it's no more harmful to propagate any error at this point. - T::Currency::repatriate_reserved(&who, &reporter, T::VotingBond::get())?; + T::Currency::repatriate_reserved(&who, &reporter, T::VotingBond::get(), BalanceStatus::Free)?; Self::deposit_event(RawEvent::VoterReaped(who, reporter)); } else { let imbalance = T::Currency::slash_reserved(&reporter, T::VotingBond::get()).0; @@ -525,7 +525,7 @@ decl_module! { let who = ensure_signed(origin)?; ensure!(!Self::presentation_active(), Error::::CannotRetractPresenting); - ensure!(>::exists(&who), Error::::RetractNonVoter); + ensure!(>::contains_key(&who), Error::::RetractNonVoter); let index = index as usize; let voter = Self::voter_at(index).ok_or(Error::::InvalidRetractionIndex)?; ensure!(voter == who, Error::::InvalidRetractionIndex); @@ -730,7 +730,7 @@ impl Module { /// If `who` a candidate at the moment? pub fn is_a_candidate(who: &T::AccountId) -> bool { - >::exists(who) + >::contains_key(who) } /// Iff the member `who` still has a seat at blocknumber `n` returns `true`. diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index 3274dd57b4afa4503b47eee183dc298998f4a558..b82e73d512aa6eb254a1eda22ff801b7ad768a95 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -39,9 +39,9 @@ parameter_types! { } impl frame_system::Trait for Test { type Origin = Origin; + type Call = (); type Index = u64; type BlockNumber = u64; - type Call = (); type Hash = H256; type Hashing = BlakeTwo256; type AccountId = u64; @@ -54,21 +54,20 @@ impl frame_system::Trait for Test { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; - pub const CreationFee: u64 = 0; + pub const ExistentialDeposit: u64 = 1; } impl pallet_balances::Trait for Test { type Balance = u64; - type OnNewAccount = (); - type OnReapAccount = System; - type Event = Event; - type TransferPayment = (); type DustRemoval = (); + type Event = Event; type ExistentialDeposit = ExistentialDeposit; - type CreationFee = CreationFee; + type AccountStore = System; } parameter_types! { @@ -151,7 +150,7 @@ frame_support::construct_runtime!( NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Module, Call, Event}, + System: system::{Module, Call, Event}, Balances: pallet_balances::{Module, Call, Event, Config}, Elections: elections::{Module, Call, Event, Config}, } @@ -266,8 +265,7 @@ pub(crate) fn create_candidate(i: u64, index: u32) { } pub(crate) fn balances(who: &u64) -> (u64, u64) { - let a = Balances::account(who); - (a.free, a.reserved) + (Balances::free_balance(who), Balances::reserved_balance(who)) } pub(crate) fn locks(who: &u64) -> Vec { diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index b45bf6e31e561a7b32169e748c7a112edadede69..d521b87270a35cd6ed17de4c26212cfe634adb90 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -1,24 +1,26 @@ [package] name = "pallet-evm" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../balances" } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -primitive-types = { version = "0.6", default-features = false, features = ["rlp"] } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-alpha.1", default-features = false, path = "../timestamp" } +pallet-balances = { version = "2.0.0-alpha.1", default-features = false, path = "../balances" } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +primitive-types = { version = "0.6.2", default-features = false, features = ["rlp"] } rlp = { version = "0.4", default-features = false } -evm = { version = "0.14", default-features = false } +evm = { version = "0.15", default-features = false } sha3 = { version = "0.8", default-features = false } [features] diff --git a/frame/evm/src/backend.rs b/frame/evm/src/backend.rs index b6c3078a3339a2d27c43c08a732599ae9972e84f..d72c8b785e3468762054c9bd028fe3e3a1b730db 100644 --- a/frame/evm/src/backend.rs +++ b/frame/evm/src/backend.rs @@ -29,7 +29,7 @@ pub struct Log { pub address: H160, /// Topics of the log. pub topics: Vec, - /// Bytearray data of the log. + /// Byte array data of the log. pub data: Vec, } diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs index edd3d46a9a2c8858319bf8cef7348b28535dc8ba..abf20114f2c6a7f2fb70760ca0de6ee3c45d26c7 100644 --- a/frame/evm/src/lib.rs +++ b/frame/evm/src/lib.rs @@ -25,7 +25,7 @@ pub use crate::backend::{Account, Log, Vicinity, Backend}; use sp_std::{vec::Vec, marker::PhantomData}; use frame_support::{ensure, decl_module, decl_storage, decl_event, decl_error}; -use frame_support::weights::{Weight, WeighData, ClassifyDispatch, DispatchClass, PaysFee}; +use frame_support::weights::{Weight, DispatchClass, FunctionOf}; use frame_support::traits::{Currency, WithdrawReason, ExistenceRequirement}; use frame_system::{self as system, ensure_signed}; use sp_runtime::ModuleId; @@ -34,6 +34,7 @@ use sp_core::{U256, H256, H160, Hasher}; use sp_runtime::{ DispatchResult, traits::{UniqueSaturatedInto, AccountIdConversion, SaturatedConversion}, }; +use sha3::{Digest, Keccak256}; use evm::{ExitReason, ExitSucceed, ExitError}; use evm::executor::StackExecutor; use evm::backend::ApplyBackend; @@ -115,38 +116,6 @@ impl Precompiles for () { } } -struct WeightForCallCreate; - -impl WeighData<(&H160, &Vec, &U256, &u32, &U256)> for WeightForCallCreate { - fn weigh_data( - &self, - (_, _, _, gas_provided, gas_price): (&H160, &Vec, &U256, &u32, &U256) - ) -> Weight { - (*gas_price).saturated_into::().saturating_mul(*gas_provided) - } -} - -impl WeighData<(&Vec, &U256, &u32, &U256)> for WeightForCallCreate { - fn weigh_data( - &self, - (_, _, gas_provided, gas_price): (&Vec, &U256, &u32, &U256) - ) -> Weight { - (*gas_price).saturated_into::().saturating_mul(*gas_provided) - } -} - -impl ClassifyDispatch for WeightForCallCreate { - fn classify_dispatch(&self, _: T) -> DispatchClass { - DispatchClass::Normal - } -} - -impl PaysFee for WeightForCallCreate { - fn pays_fee(&self, _: T) -> bool { - true - } -} - /// EVM module trait pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { /// Calculator for current gas price. @@ -197,6 +166,8 @@ decl_error! { ExitReasonRevert, /// Call returned VM fatal error ExitReasonFatal, + /// Nonce is invalid + InvalidNonce, } } @@ -206,7 +177,7 @@ decl_module! { fn deposit_event() = default; - /// Despoit balance from currency/balances module into EVM. + /// Deposit balance from currency/balances module into EVM. #[weight = SimpleDispatchInfo::FixedNormal(10_000)] fn deposit_balance(origin, value: BalanceOf) { let sender = ensure_signed(origin)?; @@ -250,7 +221,7 @@ decl_module! { } /// Issue an EVM call operation. This is similar to a message call transaction in Ethereum. - #[weight = WeightForCallCreate] + #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&H160, &Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] fn call( origin, target: H160, @@ -258,117 +229,99 @@ decl_module! { value: U256, gas_limit: u32, gas_price: U256, + nonce: Option, ) -> DispatchResult { let sender = ensure_signed(origin)?; - ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); let source = T::ConvertAccountId::convert_account_id(&sender); - let vicinity = Vicinity { - gas_price, - origin: source, - }; - - let mut backend = Backend::::new(&vicinity); - let mut executor = StackExecutor::new_with_precompile( - &backend, - gas_limit as usize, - &backend::GASOMETER_CONFIG, - T::Precompiles::execute, - ); - - let total_fee = gas_price.checked_mul(U256::from(gas_limit)) - .ok_or(Error::::FeeOverflow)?; - if Accounts::get(&source).balance < - value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)? - { - Err(Error::::BalanceLow)? - } - executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; - - let reason = executor.transact_call( + Self::execute_evm( source, - target, value, - input, - gas_limit as usize, - ); - - let ret = match reason { - ExitReason::Succeed(_) => Ok(()), - ExitReason::Error(_) => Err(Error::::ExitReasonFailed), - ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), - ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), - }; - let actual_fee = executor.fee(gas_price); - executor.deposit(source, total_fee.saturating_sub(actual_fee)); - - let (values, logs) = executor.deconstruct(); - backend.apply(values, logs, true); - - ret.map_err(Into::into) + gas_limit, + gas_price, + nonce, + |executor| ((), executor.transact_call( + source, + target, + value, + input, + gas_limit as usize, + )), + ).map_err(Into::into) } /// Issue an EVM create operation. This is similar to a contract creation transaction in /// Ethereum. - #[weight = WeightForCallCreate] + #[weight = FunctionOf(|(_, _, gas_limit, gas_price, _): (&Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] fn create( origin, init: Vec, value: U256, gas_limit: u32, gas_price: U256, + nonce: Option, ) -> DispatchResult { let sender = ensure_signed(origin)?; - ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); - let source = T::ConvertAccountId::convert_account_id(&sender); - let vicinity = Vicinity { - gas_price, - origin: source, - }; - - let mut backend = Backend::::new(&vicinity); - let mut executor = StackExecutor::new_with_precompile( - &backend, - gas_limit as usize, - &backend::GASOMETER_CONFIG, - T::Precompiles::execute, - ); - - let total_fee = gas_price.checked_mul(U256::from(gas_limit)) - .ok_or(Error::::FeeOverflow)?; - if Accounts::get(&source).balance < - value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)? - { - Err(Error::::BalanceLow)? - } - executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; - - let create_address = executor.create_address(source, evm::CreateScheme::Dynamic); - let reason = executor.transact_create( + let create_address = Self::execute_evm( source, value, - init, - gas_limit as usize, - ); - - let ret = match reason { - ExitReason::Succeed(_) => { - Module::::deposit_event(Event::Created(create_address)); - Ok(()) + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Legacy { caller: source }, + ), executor.transact_create( + source, + value, + init, + gas_limit as usize, + )) }, - ExitReason::Error(_) => Err(Error::::ExitReasonFailed), - ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), - ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), - }; - let actual_fee = executor.fee(gas_price); - executor.deposit(source, total_fee.saturating_sub(actual_fee)); + )?; - let (values, logs) = executor.deconstruct(); - backend.apply(values, logs, true); + Module::::deposit_event(Event::Created(create_address)); + Ok(()) + } - ret.map_err(Into::into) + /// Issue an EVM create2 operation. + #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&Vec, &H256, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] + fn create2( + origin, + init: Vec, + salt: H256, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option, + ) -> DispatchResult { + let sender = ensure_signed(origin)?; + let source = T::ConvertAccountId::convert_account_id(&sender); + + let code_hash = H256::from_slice(Keccak256::digest(&init).as_slice()); + let create_address = Self::execute_evm( + source, + value, + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Create2 { caller: source, code_hash, salt }, + ), executor.transact_create2( + source, + value, + init, + salt, + gas_limit as usize, + )) + }, + )?; + + Module::::deposit_event(Event::Created(create_address)); + Ok(()) } } } @@ -405,4 +358,59 @@ impl Module { AccountCodes::remove(address); AccountStorages::remove_prefix(address); } + + /// Execute an EVM operation. + fn execute_evm( + source: H160, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option, + f: F, + ) -> Result> where + F: FnOnce(&mut StackExecutor>) -> (R, ExitReason), + { + ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); + + let vicinity = Vicinity { + gas_price, + origin: source, + }; + + let mut backend = Backend::::new(&vicinity); + let mut executor = StackExecutor::new_with_precompile( + &backend, + gas_limit as usize, + &backend::GASOMETER_CONFIG, + T::Precompiles::execute, + ); + + let total_fee = gas_price.checked_mul(U256::from(gas_limit)) + .ok_or(Error::::FeeOverflow)?; + let total_payment = value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)?; + let source_account = Accounts::get(&source); + ensure!(source_account.balance >= total_payment, Error::::BalanceLow); + executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; + + if let Some(nonce) = nonce { + ensure!(source_account.nonce == nonce, Error::::InvalidNonce); + } + + let (retv, reason) = f(&mut executor); + + let ret = match reason { + ExitReason::Succeed(_) => Ok(retv), + ExitReason::Error(_) => Err(Error::::ExitReasonFailed), + ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), + ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), + }; + + let actual_fee = executor.fee(gas_price); + executor.deposit(source, total_fee.saturating_sub(actual_fee)); + + let (values, logs) = executor.deconstruct(); + backend.apply(values, logs, true); + + ret + } } diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/example-offchain-worker/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..b10d9cb390615c3fa275bc151ffdd4bd0771e100 --- /dev/null +++ b/frame/example-offchain-worker/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "pallet-example-offchain-worker" +version = "2.0.0-alpha.1" +authors = ["Parity Technologies "] +edition = "2018" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +serde = { version = "1.0.101", optional = true } +sp-core = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +serde_json = { version = "1.0.46", default-features = false, features = ["alloc"] } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "serde", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1c72a8be68653974803ee7702ae3aa53accdd1b5 --- /dev/null +++ b/frame/example-offchain-worker/src/lib.rs @@ -0,0 +1,548 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Offchain Worker Example Module +//! +//! The Offchain Worker Example: A simple pallet demonstrating +//! concepts, APIs and structures common to most offchain workers. +//! +//! Run `cargo doc --package pallet-example-offchain-worker --open` to view this module's +//! documentation. +//! +//! - \[`pallet_example_offchain_worker::Trait`](./trait.Trait.html) +//! - \[`Call`](./enum.Call.html) +//! - \[`Module`](./struct.Module.html) +//! +//! +//! \## Overview +//! +//! In this example we are going to build a very simplistic, naive and definitely NOT +//! production-ready oracle for BTC/USD price. +//! Offchain Worker (OCW) will be triggered after every block, fetch the current price +//! and prepare either signed or unsigned transaction to feed the result back on chain. +//! The on-chain logic will simply aggregate the results and store last `64` values to compute +//! the average price. +//! Additional logic in OCW is put in place to prevent spamming the network with both signed +//! and unsigned transactions, and custom `UnsignedValidator` makes sure that there is only +//! one unsigned transaction floating in the network. +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::{ + debug, + dispatch::DispatchResult, decl_module, decl_storage, decl_event, + traits::Get, + weights::SimpleDispatchInfo, +}; +use frame_system::{self as system, ensure_signed, ensure_none, offchain}; +use serde_json as json; +use sp_core::crypto::KeyTypeId; +use sp_runtime::{ + offchain::{http, Duration, storage::StorageValueRef}, + traits::Zero, + transaction_validity::{InvalidTransaction, ValidTransaction, TransactionValidity}, +}; + +#[cfg(test)] +mod tests; + +/// Defines application identifier for crypto keys of this module. +/// +/// Every module that deals with signatures needs to declare its unique identifier for +/// its crypto keys. +/// When offchain worker is signing transactions it's going to request keys of type +/// `KeyTypeId` from the keystore and use the ones it finds to sign the transaction. +/// The keys can be inserted manually via RPC (see `author_insertKey`). +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"btc!"); + +/// Based on the above `KeyTypeId` we need to generate a pallet-specific crypto type wrappers. +/// We can use from supported crypto kinds (`sr25519`, `ed25519` and `ecdsa`) and augment +/// the types with this pallet-specific identifier. +pub mod crypto { + use super::KEY_TYPE; + use sp_runtime::app_crypto::{app_crypto, sr25519}; + app_crypto!(sr25519, KEY_TYPE); +} + +/// This pallet's configuration trait +pub trait Trait: frame_system::Trait { + /// The type to sign and submit transactions. + type SubmitSignedTransaction: + offchain::SubmitSignedTransaction::Call>; + /// The type to submit unsigned transactions. + type SubmitUnsignedTransaction: + offchain::SubmitUnsignedTransaction::Call>; + + /// The overarching event type. + type Event: From> + Into<::Event>; + /// The overarching dispatch call type. + type Call: From>; + + // Configuration parameters + + /// A grace period after we send transaction. + /// + /// To avoid sending too many transactions, we only attempt to send one + /// every `GRACE_PERIOD` blocks. We use Local Storage to coordinate + /// sending between distinct runs of this offchain worker. + type GracePeriod: Get; + + /// Number of blocks of cooldown after unsigned transaction is included. + /// + /// This ensures that we only accept unsigned transactions once, every `UnsignedInterval` blocks. + type UnsignedInterval: Get; +} + +decl_storage! { + trait Store for Module as Example { + /// A vector of recently submitted prices. + /// + /// This is used to calculate average price, should have bounded size. + Prices get(fn prices): Vec; + /// Defines the block when next unsigned transaction will be accepted. + /// + /// To prevent spam of unsigned (and unpayed!) transactions on the network, + /// we only allow one transaction every `T::UnsignedInterval` blocks. + /// This storage entry defines when new transaction is going to be accepted. + NextUnsignedAt get(fn next_unsigned_at): T::BlockNumber; + } +} + +decl_event!( + /// Events generated by the module. + pub enum Event where AccountId = ::AccountId { + /// Event generated when new price is accepted to contribute to the average. + NewPrice(u32, AccountId), + } +); + +decl_module! { + /// A public part of the pallet. + pub struct Module for enum Call where origin: T::Origin { + fn deposit_event() = default; + + /// Submit new price to the list. + /// + /// This method is a public function of the module and can be called from within + /// a transaction. It appends given `price` to current list of prices. + /// In our example the `offchain worker` will create, sign & submit a transaction that + /// calls this function passing the price. + /// + /// The transaction needs to be signed (see `ensure_signed`) check, so that the caller + /// pays a fee to execute it. + /// This makes sure that it's not easy (or rather cheap) to attack the chain by submitting + /// excesive transactions, but note that it doesn't ensure the price oracle is actually + /// working and receives (and provides) meaningful data. + /// This example is not focused on correctness of the oracle itself, but rather its + /// purpose is to showcase offchain worker capabilities. + #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + pub fn submit_price(origin, price: u32) -> DispatchResult { + // Retrieve sender of the transaction. + let who = ensure_signed(origin)?; + // Add the price to the on-chain list. + Self::add_price(who, price); + Ok(()) + } + + /// Submit new price to the list via unsigned transaction. + /// + /// Works exactly like the `submit_price` function, but since we allow sending the + /// transaction without a signature, and hence without paying any fees, + /// we need a way to make sure that only some transactions are accepted. + /// This function can be called only once every `T::UnsignedInterval` blocks. + /// Transactions that call that function are de-duplicated on the pool level + /// via `validate_unsigned` implementation and also are rendered invalid if + /// the function has already been called in current "session". + /// + /// It's important to specify `weight` for unsigned calls as well, because even though + /// they don't charge fees, we still don't want a single block to contain unlimited + /// number of such transactions. + /// + /// This example is not focused on correctness of the oracle itself, but rather its + /// purpose is to showcase offchain worker capabilities. + #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + pub fn submit_price_unsigned(origin, _block_number: T::BlockNumber, price: u32) + -> DispatchResult + { + // This ensures that the function can only be called via unsigned transaction. + ensure_none(origin)?; + // Add the price to the on-chain list, but mark it as coming from an empty address. + Self::add_price(Default::default(), price); + // now increment the block number at which we expect next unsigned transaction. + let current_block = >::block_number(); + >::put(current_block + T::UnsignedInterval::get()); + Ok(()) + } + + /// Offchain Worker entry point. + /// + /// By implementing `fn offchain_worker` within `decl_module!` you declare a new offchain + /// worker. + /// This function will be called when the node is fully synced and a new best block is + /// succesfuly imported. + /// Note that it's not guaranteed for offchain workers to run on EVERY block, there might + /// be cases where some blocks are skipped, or for some the worker runs twice (re-orgs), + /// so the code should be able to handle that. + /// You can use `Local Storage` API to coordinate runs of the worker. + fn offchain_worker(block_number: T::BlockNumber) { + // It's a good idea to add logs to your offchain workers. + // Using the `frame_support::debug` module you have access to the same API exposed by + // the `log` crate. + // Note that having logs compiled to WASM may cause the size of the blob to increase + // significantly. You can use `RuntimeDebug` custom derive to hide details of the types + // in WASM or use `debug::native` namespace to produce logs only when the worker is + // running natively. + debug::native::info!("Hello World from offchain workers!"); + + // Since off-chain workers are just part of the runtime code, they have direct access + // to the storage and other included pallets. + // + // We can easily import `frame_system` and retrieve a block hash of the parent block. + let parent_hash = >::block_hash(block_number - 1.into()); + debug::debug!("Current block: {:?} (parent hash: {:?})", block_number, parent_hash); + + // It's a good practice to keep `fn offchain_worker()` function minimal, and move most + // of the code to separate `impl` block. + // Here we call a helper function to calculate current average price. + // This function reads storage entries of the current state. + let average: Option = Self::average_price(); + debug::debug!("Current price: {:?}", average); + + // For this example we are going to send both signed and unsigned transactions + // depending on the block number. + // Usually it's enough to choose one or the other. + let should_send = Self::choose_transaction_type(block_number); + let res = match should_send { + TransactionType::Signed => Self::fetch_price_and_send_signed(), + TransactionType::Unsigned => Self::fetch_price_and_send_unsigned(block_number), + TransactionType::None => Ok(()), + }; + if let Err(e) = res { + debug::error!("Error: {}", e); + } + } + } +} + +enum TransactionType { + Signed, + Unsigned, + None, +} + +/// Most of the functions are moved outside of the `decl_module!` macro. +/// +/// This greatly helps with error messages, as the ones inside the macro +/// can sometimes be hard to debug. +impl Module { + /// Chooses which transaction type to send. + /// + /// This function serves mostly to showcase `StorageValue` helper + /// and local storage usage. + /// + /// Returns a type of transaction that should be produced in current run. + fn choose_transaction_type(block_number: T::BlockNumber) -> TransactionType { + /// A friendlier name for the error that is going to be returned in case we are in the grace + /// period. + const RECENTLY_SENT: () = (); + + // Start off by creating a reference to Local Storage value. + // Since the local storage is common for all offchain workers, it's a good practice + // to prepend your entry with the module name. + let val = StorageValueRef::persistent(b"example_ocw::last_send"); + // The Local Storage is persisted and shared between runs of the offchain workers, + // and offchain workers may run concurrently. We can use the `mutate` function, to + // write a storage entry in an atomic fashion. Under the hood it uses `compare_and_set` + // low-level method of local storage API, which means that only one worker + // will be able to "acquire a lock" and send a transaction if multiple workers + // happen to be executed concurrently. + let res = val.mutate(|last_send: Option>| { + // We match on the value decoded from the storage. The first `Option` + // indicates if the value was present in the storage at all, + // the second (inner) `Option` indicates if the value was succesfuly + // decoded to expected type (`T::BlockNumber` in our case). + match last_send { + // If we already have a value in storage and the block number is recent enough + // we avoid sending another transaction at this time. + Some(Some(block)) if block + T::GracePeriod::get() < block_number => { + Err(RECENTLY_SENT) + }, + // In every other case we attempt to acquire the lock and send a transaction. + _ => Ok(block_number) + } + }); + + // The result of `mutate` call will give us a nested `Result` type. + // The first one matches the return of the closure passed to `mutate`, i.e. + // if we return `Err` from the closure, we get an `Err` here. + // In case we return `Ok`, here we will have another (inner) `Result` that indicates + // if the value has been set to the storage correctly - i.e. if it wasn't + // written to in the meantime. + match res { + // The value has been set correctly, which means we can safely send a transaction now. + Ok(Ok(block_number)) => { + // Depending if the block is even or odd we will send a `Signed` or `Unsigned` + // transaction. + // Note that this logic doesn't really guarantee that the transactions will be sent + // in an alternating fashion (i.e. fairly distributed). Depending on the execution + // order and lock acquisition, we may end up for instance sending two `Signed` + // transactions in a row. If a strict order is desired, it's better to use + // the storage entry for that. (for instance store both block number and a flag + // indicating the type of next transaction to send). + let send_signed = block_number % 2.into() == Zero::zero(); + if send_signed { + TransactionType::Signed + } else { + TransactionType::Unsigned + } + }, + // We are in the grace period, we should not send a transaction this time. + Err(RECENTLY_SENT) => TransactionType::None, + // We wanted to send a transaction, but failed to write the block number (acquire a + // lock). This indicates that another offchain worker that was running concurrently + // most likely executed the same logic and succeeded at writing to storage. + // Thus we don't really want to send the transaction, knowing that the other run + // already did. + Ok(Err(_)) => TransactionType::None, + } + } + + /// A helper function to fetch the price and send signed transaction. + fn fetch_price_and_send_signed() -> Result<(), String> { + use system::offchain::SubmitSignedTransaction; + // Firstly we check if there are any accounts in the local keystore that are capable of + // signing the transaction. + // If not it doesn't even make sense to make external HTTP requests, since we won't be able + // to put the results back on-chain. + if !T::SubmitSignedTransaction::can_sign() { + return Err( + "No local accounts available. Consider adding one via `author_insertKey` RPC." + )? + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?; + + // Received price is wrapped into a call to `submit_price` public function of this pallet. + // This means that the transaction, when executed, will simply call that function passing + // `price` as an argument. + let call = Call::submit_price(price); + + // Using `SubmitSignedTransaction` associated type we create and submit a transaction + // representing the call, we've just created. + // Submit signed will return a vector of results for all accounts that were found in the + // local keystore with expected `KEY_TYPE`. + let results = T::SubmitSignedTransaction::submit_signed(call); + for (acc, res) in &results { + match res { + Ok(()) => debug::info!("[{:?}] Submitted price of {} cents", acc, price), + Err(e) => debug::error!("[{:?}] Failed to submit transaction: {:?}", acc, e), + } + } + + Ok(()) + } + + /// A helper function to fetch the price and send unsigned transaction. + fn fetch_price_and_send_unsigned(block_number: T::BlockNumber) -> Result<(), String> { + use system::offchain::SubmitUnsignedTransaction; + // Make sure we don't fetch the price if unsigned transaction is going to be rejected + // anyway. + let next_unsigned_at = >::get(); + if next_unsigned_at > block_number { + return Err( + format!("Too early to send unsigned transaction. Next at: {:?}", next_unsigned_at) + )? + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?; + + // Received price is wrapped into a call to `submit_price_unsigned` public function of this + // pallet. This means that the transaction, when executed, will simply call that function + // passing `price` as an argument. + let call = Call::submit_price_unsigned(block_number, price); + + // Now let's create an unsigned transaction out of this call and submit it to the pool. + // By default unsigned transactions are disallowed, so we need to whitelist this case + // by writing `UnsignedValidator`. Note that it's EXTREMELY important to carefuly + // implement unsigned validation logic, as any mistakes can lead to opening DoS or spam + // attack vectors. See validation logic docs for more details. + T::SubmitUnsignedTransaction::submit_unsigned(call) + .map_err(|()| "Unable to submit unsigned transaction.".into()) + + } + + /// Fetch current price and return the result in cents. + fn fetch_price() -> Result { + // We want to keep the offchain worker execution time reasonable, so we set a hard-coded + // deadline to 2s to complete the external call. + // You can also wait idefinitely for the response, however you may still get a timeout + // coming from the host machine. + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(2_000)); + // Initiate an external HTTP GET request. + // This is using high-level wrappers from `sp_runtime`, for the low-level calls that + // you can find in `sp_io`. The API is trying to be similar to `reqwest`, but + // since we are running in a custom WASM execution environment we can't simply + // import the library here. + let request = http::Request::get( + "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD" + ); + // We set the deadline for sending of the request, note that awaiting response can + // have a separate deadline. Next we send the request, before that it's also possible + // to alter request headers or stream body content in case of non-GET requests. + let pending = request + .deadline(deadline) + .send() + .map_err(|_| http::Error::IoError)?; + + // The request is already being processed by the host, we are free to do anything + // else in the worker (we can send multiple concurrent requests too). + // At some point however we probably want to check the response though, + // so we can block current thread and wait for it to finish. + // Note that since the request is being driven by the host, we don't have to wait + // for the request to have it complete, we will just not read the response. + let response = pending.try_wait(deadline) + .map_err(|_| http::Error::DeadlineReached)??; + // Let's check the status code before we proceed to reading the response. + if response.code != 200 { + debug::warn!("Unexpected status code: {}", response.code); + return Err(http::Error::Unknown); + } + + // Next we want to fully read the response body and collect it to a vector of bytes. + // Note that the return object allows you to read the body in chunks as well + // with a way to control the deadline. + let body = response.body().collect::>(); + // Next we parse the response using `serde_json`. Even though it's possible to use + // `serde_derive` and deserialize to a struct it's not recommended due to blob size + // overhead introduced by such code. Deserializing to `json::Value` is much more + // lightweight and should be preferred, especially if we only care about a small number + // of properties from the response. + let val: Result = json::from_slice(&body); + // Let's parse the price as float value. Note that you should avoid using floats in the + // runtime, it's fine to do that in the offchain worker, but we do convert it to an integer + // before submitting on-chain. + let price = val.ok().and_then(|v| v.get("USD").and_then(|v| v.as_f64())); + let price = match price { + Some(pricef) => Ok((pricef * 100.) as u32), + None => { + let s = core::str::from_utf8(&body); + debug::warn!("Unable to extract price from the response: {:?}", s); + Err(http::Error::Unknown) + } + }?; + + debug::warn!("Got price: {} cents", price); + + Ok(price) + } + + /// Add new price to the list. + fn add_price(who: T::AccountId, price: u32) { + debug::info!("Adding to the average: {}", price); + Prices::mutate(|prices| { + const MAX_LEN: usize = 64; + + if prices.len() < MAX_LEN { + prices.push(price); + } else { + prices[price as usize % MAX_LEN] = price; + } + }); + + let average = Self::average_price() + .expect("The average is not empty, because it was just mutated; qed"); + debug::info!("Current average price is: {}", average); + // here we are raising the NewPrice event + Self::deposit_event(RawEvent::NewPrice(price, who)); + } + + /// Calculate current average price. + fn average_price() -> Option { + let prices = Prices::get(); + if prices.is_empty() { + None + } else { + Some(prices.iter().fold(0_u32, |a, b| a.saturating_add(*b)) / prices.len() as u32) + } + } +} + +#[allow(deprecated)] // ValidateUnsigned +impl frame_support::unsigned::ValidateUnsigned for Module { + type Call = Call; + + /// Validate unsigned call to this module. + /// + /// By default unsigned transactions are disallowed, but implementing the validator + /// here we make sure that some particular calls (the ones produced by offchain worker) + /// are being whitelisted and marked as valid. + fn validate_unsigned(call: &Self::Call) -> TransactionValidity { + // Firstly let's check that we call the right function. + if let Call::submit_price_unsigned(block_number, new_price) = call { + // Now let's check if the transaction has any chance to succeed. + let next_unsigned_at = >::get(); + if &next_unsigned_at > block_number { + return InvalidTransaction::Stale.into(); + } + // Let's make sure to reject transactions from the future. + let current_block = >::block_number(); + if ¤t_block < block_number { + return InvalidTransaction::Future.into(); + } + + // We prioritize transactions that are more far away from current average. + // + // Note this doesn't make much sense when building an actual oracle, but this example + // is here mostly to show off offchain workers capabilities, not about building an + // oracle. + let avg_price = Self::average_price() + .map(|price| if &price > new_price { price - new_price } else { new_price - price }) + .unwrap_or(0); + + Ok(ValidTransaction { + // We set base priority to 2**20 to make sure it's included before any other + // transactions in the pool. Next we tweak the priority depending on how much + // it differs from the current average. (the more it differs the more priority it + // has). + priority: (1 << 20) + avg_price as u64, + // This transaction does not require anything else to go before into the pool. + // In theory we could require `previous_unsigned_at` transaction to go first, + // but it's not necessary in our case. + requires: vec![], + // We set the `provides` tag to be the same as `next_unsigned_at`. This makes + // sure only one transaction produced after `next_unsigned_at` will ever + // get to the transaction pool and will end up in the block. + // We can still have multiple transactions compete for the same "spot", + // and the one with higher priority will replace other one in the pool. + provides: vec![codec::Encode::encode(&(KEY_TYPE.0, next_unsigned_at))], + // The transaction is only valid for next 5 blocks. After that it's + // going to be revalidated by the pool. + longevity: 5, + // It's fine to propagate that transaction to other peers, which means it can be + // created even by nodes that don't produce blocks. + // Note that sometimes it's better to keep it for yourself (if you are the block + // producer), since for instance in some schemes others may copy your solution and + // claim a reward. + propagate: true, + }) + } else { + InvalidTransaction::Call.into() + } + } +} diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/example-offchain-worker/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b6a567a178403d9cb1b717705fa520820358bef --- /dev/null +++ b/frame/example-offchain-worker/src/tests.rs @@ -0,0 +1,210 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::*; + +use codec::Decode; +use frame_support::{ + assert_ok, impl_outer_origin, parameter_types, + weights::{GetDispatchInfo, Weight}, +}; +use sp_core::{ + H256, + offchain::{OffchainExt, TransactionPoolExt, testing}, + testing::KeyStore, + traits::KeystoreExt, +}; +use sp_runtime::{ + Perbill, RuntimeAppPublic, + testing::{Header, TestXt}, + traits::{BlakeTwo256, IdentityLookup, Extrinsic as ExtrinsicsT}, +}; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +// For testing the module, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of modules we want to use. +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type Origin = Origin; + type Call = (); + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = sp_core::sr25519::Public; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); +} + +type Extrinsic = TestXt, ()>; +type SubmitTransaction = frame_system::offchain::TransactionSubmitter< + crypto::Public, + Test, + Extrinsic +>; + +impl frame_system::offchain::CreateTransaction for Test { + type Public = sp_core::sr25519::Public; + type Signature = sp_core::sr25519::Signature; + + fn create_transaction>( + call: ::Call, + _public: Self::Public, + _account: ::AccountId, + nonce: ::Index, + ) -> Option<(::Call, ::SignaturePayload)> { + Some((call, (nonce, ()))) + } +} + +parameter_types! { + pub const GracePeriod: u64 = 5; + pub const UnsignedInterval: u64 = 128; +} + +impl Trait for Test { + type Event = (); + type Call = Call; + type SubmitSignedTransaction = SubmitTransaction; + type SubmitUnsignedTransaction = SubmitTransaction; + type GracePeriod = GracePeriod; + type UnsignedInterval = UnsignedInterval; +} + +type Example = Module; + +#[test] +fn it_aggregates_the_price() { + sp_io::TestExternalities::default().execute_with(|| { + assert_eq!(Example::average_price(), None); + + assert_ok!(Example::submit_price(Origin::signed(Default::default()), 27)); + assert_eq!(Example::average_price(), Some(27)); + + assert_ok!(Example::submit_price(Origin::signed(Default::default()), 43)); + assert_eq!(Example::average_price(), Some(35)); + }); +} + +#[test] +fn should_make_http_call_and_parse_result() { + let (offchain, state) = testing::TestOffchainExt::new(); + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + + price_oracle_response(&mut state.write()); + + t.execute_with(|| { + // when + let price = Example::fetch_price().unwrap(); + // then + assert_eq!(price, 15522); + }); +} + +#[test] +fn should_submit_signed_transaction_on_chain() { + const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten"; + + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let keystore = KeyStore::new(); + keystore.write().sr25519_generate_new( + crate::crypto::Public::ID, + Some(&format!("{}/hunter1", PHRASE)) + ).unwrap(); + + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + t.register_extension(KeystoreExt(keystore)); + + price_oracle_response(&mut offchain_state.write()); + + t.execute_with(|| { + // when + Example::fetch_price_and_send_signed().unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature.unwrap().0, 0); + assert_eq!(tx.call, Call::submit_price(15522)); + }); +} + +#[test] +fn should_submit_unsigned_transaction_on_chain() { + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + + price_oracle_response(&mut offchain_state.write()); + + t.execute_with(|| { + // when + Example::fetch_price_and_send_unsigned(1).unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature, None); + assert_eq!(tx.call, Call::submit_price_unsigned(1, 15522)); + }); +} + +#[test] +fn weights_work() { + // must have a default weight. + let default_call = >::submit_price(10); + let info = default_call.get_dispatch_info(); + // aka. `let info = as GetDispatchInfo>::get_dispatch_info(&default_call);` + assert_eq!(info.weight, 10_000); +} + +fn price_oracle_response(state: &mut testing::OffchainState) { + state.expect_request(0, testing::PendingRequest { + method: "GET".into(), + uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), + response: Some(br#"{"USD": 155.23}"#.to_vec()), + sent: true, + ..Default::default() + }); +} diff --git a/frame/example/Cargo.toml b/frame/example/Cargo.toml index 0d37940fc73170feaae1eace009531ea9fb692d7..bb6625e8925076e3113f05c8be16f46676cf4a0d 100644 --- a/frame/example/Cargo.toml +++ b/frame/example/Cargo.toml @@ -1,22 +1,24 @@ [package] name = "pallet-example" -version = "2.0.0" +version = "2.0.0-alpha.1" authors = ["Parity Technologies "] edition = "2018" -license = "GPL-3.0" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../balances" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-alpha.1", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.1", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-alpha.1", default-features = false, path = "../balances" } +sp-runtime = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.1", default-features = false, path = "../../primitives/io" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.1", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index b3a86b6acd78e89c6f3d6bb2e4a66395a0e4ea03..f4139a310a67a90d885e66b20d234ed56f10bac1 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -14,25 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! # Example Module +//! # Example Pallet //! //! -//! The Example: A simple example of a runtime module demonstrating -//! concepts, APIs and structures common to most runtime modules. +//! The Example: A simple example of a FRAME pallet demonstrating +//! concepts, APIs and structures common to most FRAME runtimes. //! -//! Run `cargo doc --package pallet-example --open` to view this module's documentation. +//! Run `cargo doc --package pallet-example --open` to view this pallet's documentation. //! //! ### Documentation Guidelines: //! //! -//! +//! //!